granularity results granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb @ 2026-09-21T07:50:43Z from sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0
Browse files- granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/manifest.txt +19 -0
- granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/pip_freeze.txt +317 -0
- granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/provenance.txt +90 -0
- granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep1/mem_samples.csv +200 -0
- granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep1/run.env +102 -0
- granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep1/train.log +1073 -0
- granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep2/mem_samples.csv +192 -0
- granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep2/run.env +102 -0
- granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep2/train.log +1072 -0
- granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep3/mem_samples.csv +192 -0
- granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep3/run.env +102 -0
- granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep3/train.log +1072 -0
- granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/results.tsv +4 -0
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/manifest.txt
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
timestamp: 2026-09-21T07:47:26+00:00
|
| 2 |
+
host: sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0
|
| 3 |
+
gpu: NVIDIA B200 x8
|
| 4 |
+
driver: 580.95.05
|
| 5 |
+
study_scripts_branch: granularity-ultraep
|
| 6 |
+
study_scripts_commit: 05dd2cb
|
| 7 |
+
campaign: ultraep
|
| 8 |
+
backend: ultraep
|
| 9 |
+
phase: tierb
|
| 10 |
+
gbs: 8
|
| 11 |
+
seq: 4096
|
| 12 |
+
configs: G1 G2 G3 G4
|
| 13 |
+
capacity: 1.25
|
| 14 |
+
repeats: 3
|
| 15 |
+
nsys: 1
|
| 16 |
+
megatron_repo: /workspace/Megatron-UltraEP
|
| 17 |
+
megatron_branch_required: ultraep-bracing-regions-through-kernels
|
| 18 |
+
megatron_branch_actual: ultraep-bracing-regions-through-kernels
|
| 19 |
+
megatron_commit: 03d57ef7
|
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/pip_freeze.txt
ADDED
|
@@ -0,0 +1,317 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
absl-py==2.3.0
|
| 2 |
+
aiohappyeyeballs @ file:///rapids/aiohappyeyeballs-2.6.1-py3-none-any.whl#sha256=f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8
|
| 3 |
+
aiohttp @ file:///rapids/aiohttp-3.12.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=94f98e0e5a49f89b252e115844f756c04fc8050f38252a32a3dd994ce8121f10
|
| 4 |
+
aiosignal @ file:///rapids/aiosignal-1.3.2-py2.py3-none-any.whl#sha256=45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5
|
| 5 |
+
annotated-types==0.7.0
|
| 6 |
+
anyio==4.9.0
|
| 7 |
+
apex @ file:///opt/pytorch/apex/dist/apex-0.1-cp312-cp312-linux_x86_64.whl#sha256=a1508e4a386362e9c8f46c92ccada9110e421212b964c694235d0fd5a224b170
|
| 8 |
+
argon2-cffi==25.1.0
|
| 9 |
+
argon2-cffi-bindings==21.2.0
|
| 10 |
+
arrow==1.3.0
|
| 11 |
+
asciitree @ file:///rapids/asciitree-0.3.3-py3-none-any.whl#sha256=3e507ccfaeb417823158091fdb9d26b0205f6a59ba680b479a71db65c73d01d4
|
| 12 |
+
asttokens==3.0.0
|
| 13 |
+
astunparse==1.6.3
|
| 14 |
+
async-lru==2.0.5
|
| 15 |
+
attrs==25.3.0
|
| 16 |
+
audioread==3.0.1
|
| 17 |
+
babel==2.17.0
|
| 18 |
+
beautifulsoup4==4.13.4
|
| 19 |
+
black==25.1.0
|
| 20 |
+
bleach==6.2.0
|
| 21 |
+
blis==0.7.11
|
| 22 |
+
cachetools @ file:///rapids/cachetools-6.0.0-py3-none-any.whl#sha256=82e73ba88f7b30228b5507dce1a1f878498fc669d972aef2dde4f3a3c24f103e
|
| 23 |
+
catalogue==2.0.10
|
| 24 |
+
certifi==2025.4.26
|
| 25 |
+
cffi==1.17.1
|
| 26 |
+
charset-normalizer==3.4.2
|
| 27 |
+
click==8.5.0
|
| 28 |
+
cloudpathlib==0.21.1
|
| 29 |
+
cloudpickle @ file:///rapids/cloudpickle-3.1.1-py3-none-any.whl#sha256=c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e
|
| 30 |
+
cmake==3.31.6
|
| 31 |
+
comm==0.2.2
|
| 32 |
+
confection==0.1.5
|
| 33 |
+
contourpy==1.3.2
|
| 34 |
+
cuda-bindings @ file:///rapids/cuda_bindings-12.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=ff0e28d1e34758654b9c961e1f55e4786e49aee6a4dbceaf3cc24c46c672df7e
|
| 35 |
+
cuda-python @ file:///rapids/cuda_python-12.9.0-py3-none-any.whl#sha256=926acba49b2c0a0374c61b7c98f337c085199cf51cdfe4d6423c4129c20547a7
|
| 36 |
+
cudf @ file:///rapids/cudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b779a8e8c0e8577b9b560895958496da211bfac49cdc4f11a49e8d18cb82e348
|
| 37 |
+
cudf-polars @ file:///rapids/cudf_polars-25.4.0-py3-none-any.whl#sha256=227b6c5147e93ea86282c8f7219118f916769fa2ca0cdd931e97a148aa66b869
|
| 38 |
+
cugraph @ file:///rapids/cugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b837c2dc2f7b235c702e4f8df64479b8fcbb342f848516d5faf0cdc4b5f31f29
|
| 39 |
+
cugraph-service-client @ file:///rapids/cugraph_service_client-25.4.0-py3-none-any.whl#sha256=7e568c3e41154b9752f5173555ffcc282ac52d72343feeb79653b2e2b1232470
|
| 40 |
+
cugraph-service-server @ file:///rapids/cugraph_service_server-25.4.0-py3-none-any.whl#sha256=a2830dcffafcccb16e12ebf85b98f8f247869d38c01ab4dd8664a1e7439d31be
|
| 41 |
+
cuml @ file:///rapids/cuml-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1e90a464a18b094e24797f1676688c5d7696c4cf97c934504214a53d5aa09aa5
|
| 42 |
+
cupy-cuda12x @ file:///rapids/cupy_cuda12x-13.3.0-cp312-cp312-linux_x86_64.whl#sha256=6a2dfa0aa1e8189a10d1534276c08f1d2f73b747fd28bfbe7b9f4ef66d9af73f
|
| 43 |
+
cuvs @ file:///rapids/cuvs-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b12f6b9f80e053a3b38542c2947d2516e468c02cad54a13d8b1a4a61d5aa7914
|
| 44 |
+
cycler==0.12.1
|
| 45 |
+
cymem==2.0.11
|
| 46 |
+
Cython==3.1.1
|
| 47 |
+
dask @ file:///rapids/dask-2025.2.0-py3-none-any.whl#sha256=f0fdeef6ceb0a06569d456c9e704f220f7f54e80f3a6ea42ab98cea6bc642b6e
|
| 48 |
+
dask-cuda @ file:///rapids/dask_cuda-25.4.0-py3-none-any.whl#sha256=442d87da0a95d00ca2a262b64c76e785a32ad905c82000fa2227932f81390e19
|
| 49 |
+
dask-cudf @ file:///rapids/dask_cudf-25.4.0-py3-none-any.whl#sha256=5e7cea0b26daea17da6e0bb457435e24460f76ba740358a3a18e239d6015c95c
|
| 50 |
+
datasets==5.0.1
|
| 51 |
+
debugpy==1.8.14
|
| 52 |
+
decorator==5.2.1
|
| 53 |
+
-e git+https://github.com/deepseek-ai/DeepEP.git@34152ae28f80bcc3ee38d7a12cb2ad87cfd4ea72#egg=deep_ep
|
| 54 |
+
defusedxml==0.7.1
|
| 55 |
+
dill==0.4.1
|
| 56 |
+
distributed @ file:///rapids/distributed-2025.2.0-py3-none-any.whl#sha256=368462084be47092d0a835f6c64c9b852da64dc88302b76d5f93bd5131228c67
|
| 57 |
+
distributed-ucxx @ file:///rapids/distributed_ucxx-0.43.0-py3-none-any.whl#sha256=94f60d0ccb0c94ba8dbfa1eaf455874d18aaab17e2ae218c38590e9fdc2d721d
|
| 58 |
+
dm-tree==0.1.9
|
| 59 |
+
einops==0.8.1
|
| 60 |
+
execnet==2.1.1
|
| 61 |
+
executing==2.2.0
|
| 62 |
+
expecttest==0.3.0
|
| 63 |
+
fasteners @ file:///rapids/fasteners-0.19-py3-none-any.whl#sha256=758819cb5d94cdedf4e836988b74de396ceacb8e2794d21f82d131fd9ee77237
|
| 64 |
+
fastjsonschema==2.21.1
|
| 65 |
+
fastrlock @ file:///rapids/fastrlock-0.8.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl#sha256=5f13ec08f1adb1aa916c384b05ecb7dbebb8df9ea81abd045f60941c6283a670
|
| 66 |
+
filelock==3.18.0
|
| 67 |
+
flash_attn @ file:///opt/pytorch/flash_attn-2.7.4.post1-cp312-cp312-linux_x86_64.whl#sha256=a10455d6c56b387785b7413fd0bbcb392b42b3d4f4b88fda5d86cc8a9ce9182a
|
| 68 |
+
fonttools==4.58.1
|
| 69 |
+
fqdn==1.5.1
|
| 70 |
+
frozenlist @ file:///rapids/frozenlist-1.6.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=4e1be9111cb6756868ac242b3c2bd1f09d9aea09846e4f5c23715e7afb647103
|
| 71 |
+
fsspec==2025.5.1
|
| 72 |
+
gast==0.6.0
|
| 73 |
+
grpcio==1.62.1
|
| 74 |
+
h11==0.16.0
|
| 75 |
+
hf-xet==1.6.0
|
| 76 |
+
httpcore==1.0.9
|
| 77 |
+
httpx==0.28.1
|
| 78 |
+
huggingface_hub==1.32.0
|
| 79 |
+
hypothesis==6.130.8
|
| 80 |
+
idna==3.10
|
| 81 |
+
importlib_metadata==8.7.0
|
| 82 |
+
iniconfig==2.1.0
|
| 83 |
+
intel-openmp==2021.4.0
|
| 84 |
+
ipykernel==6.29.5
|
| 85 |
+
ipython==9.3.0
|
| 86 |
+
ipython_pygments_lexers==1.1.1
|
| 87 |
+
isoduration==20.11.0
|
| 88 |
+
isort==6.0.1
|
| 89 |
+
jedi==0.19.2
|
| 90 |
+
Jinja2==3.1.6
|
| 91 |
+
joblib==1.5.1
|
| 92 |
+
json5==0.12.0
|
| 93 |
+
jsonpointer==3.0.0
|
| 94 |
+
jsonschema==4.24.0
|
| 95 |
+
jsonschema-specifications==2025.4.1
|
| 96 |
+
jupyter-events==0.12.0
|
| 97 |
+
jupyter-lsp==2.2.5
|
| 98 |
+
jupyter_client==8.6.3
|
| 99 |
+
jupyter_core==5.8.1
|
| 100 |
+
jupyter_server==2.16.0
|
| 101 |
+
jupyter_server_terminals==0.5.3
|
| 102 |
+
jupyterlab==4.4.3
|
| 103 |
+
jupyterlab_code_formatter==3.0.2
|
| 104 |
+
jupyterlab_pygments==0.3.0
|
| 105 |
+
jupyterlab_server==2.27.3
|
| 106 |
+
jupyterlab_tensorboard_pro @ file:///builder/jupyterlab_tensorboard_pro-4.0.0-py2.py3-none-any.whl#sha256=6bfcb5ced3358eb72150f4b2ab91312faef2f0ca14d845e8379d8fa52df6902e
|
| 107 |
+
jupytext==1.17.2
|
| 108 |
+
kiwisolver==1.4.8
|
| 109 |
+
kvikio @ file:///rapids/kvikio-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=8b40fd8ea9e4a2974c04b50b7fc26b06094ac807f9a233281d0bafbaa71b1d85
|
| 110 |
+
langcodes==3.5.0
|
| 111 |
+
language_data==1.3.0
|
| 112 |
+
lazy_loader==0.4
|
| 113 |
+
libcudf @ file:///rapids/libcudf-25.4.0-py3-none-linux_x86_64.whl#sha256=238320b4958c1737bba2a609673e2539a78216b811b5498a68e00395d9b021a4
|
| 114 |
+
libcugraph @ file:///rapids/libcugraph-25.4.0-py3-none-linux_x86_64.whl#sha256=29085e901b2f7569ff2653bb1355fe6fad20f2fcaa585f5649da9867d8df766d
|
| 115 |
+
libcuml @ file:///rapids/libcuml-25.4.0-py3-none-linux_x86_64.whl#sha256=8e2e444c72b0e995e15a48da37ff12fb410a47a2baa69ba86ccc9b1a3db91e8c
|
| 116 |
+
libcuvs @ file:///rapids/libcuvs-25.4.0-py3-none-linux_x86_64.whl#sha256=e2f8db61c92b769c590dd752f43439ddd62b2f6592a3c337362b6c1ef220c3b9
|
| 117 |
+
libkvikio @ file:///rapids/libkvikio-25.4.0-py3-none-linux_x86_64.whl#sha256=da8a5713b05fc478bf40529a3e0fb61a2d605bb5a70ca4d461f96a6005dbe8b7
|
| 118 |
+
libraft @ file:///rapids/libraft-25.4.0-py3-none-linux_x86_64.whl#sha256=b921dec005aa4a321dd156776a18376feb47c96a5b67eec125c8a8e19edd1e24
|
| 119 |
+
librmm @ file:///rapids/librmm-25.4.0-py3-none-linux_x86_64.whl#sha256=ff951811bda729d4fa1109ab906225ae1f489a6ec2eeab5d1b674cc34bd4422f
|
| 120 |
+
librmm-cu12 @ file:///rapids/librmm_cu12-25.4.0-py3-none-any.whl#sha256=29296ef2b0e90679a0db280e86015be12d6ab9f4e62a181aa085f62dbadd06b3
|
| 121 |
+
librosa==0.11.0
|
| 122 |
+
libucxx @ file:///rapids/libucxx-0.43.0-py3-none-linux_x86_64.whl#sha256=f15849d7ac2fb68f0fd8998838a13321ddcfb0c594f7e6cbc401a3bcc4750566
|
| 123 |
+
lightning-thunder @ file:///opt/pytorch/lightning-thunder
|
| 124 |
+
lightning-utilities==0.14.3
|
| 125 |
+
lintrunner==0.12.7
|
| 126 |
+
llvmlite==0.42.0
|
| 127 |
+
locket @ file:///rapids/locket-1.0.0-py2.py3-none-any.whl#sha256=b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3
|
| 128 |
+
looseversion==1.3.0
|
| 129 |
+
marisa-trie==1.2.1
|
| 130 |
+
Markdown==3.8
|
| 131 |
+
markdown-it-py==3.0.0
|
| 132 |
+
MarkupSafe==3.0.2
|
| 133 |
+
matplotlib==3.10.3
|
| 134 |
+
matplotlib-inline==0.1.7
|
| 135 |
+
mdit-py-plugins==0.4.2
|
| 136 |
+
mdurl==0.1.2
|
| 137 |
+
-e git+ssh://git@github.com/sriram-17-17/my-Megatron-MoE-Echo.git@9a23679390d7ab1fe462bc61fa31d90d393db49c#egg=megatron_core
|
| 138 |
+
mistune==3.1.3
|
| 139 |
+
mkl==2021.1.1
|
| 140 |
+
mkl-devel==2021.1.1
|
| 141 |
+
mkl-include==2021.1.1
|
| 142 |
+
ml_dtypes==0.6.0
|
| 143 |
+
mock==5.2.0
|
| 144 |
+
-e git+ssh://git@github.com/sriram-17-17/MoonEP.git@d8bc6667fad7177e6e09b637d773fa67f08bae93#egg=moonep
|
| 145 |
+
mpmath==1.3.0
|
| 146 |
+
msgpack==1.1.0
|
| 147 |
+
multidict @ file:///rapids/multidict-6.4.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=a920f9cf2abdf6e493c519492d892c362007f113c94da4c239ae88429835bad1
|
| 148 |
+
multiprocess==0.70.19
|
| 149 |
+
murmurhash==1.0.13
|
| 150 |
+
mypy_extensions==1.1.0
|
| 151 |
+
nbclient==0.10.2
|
| 152 |
+
nbconvert==7.16.6
|
| 153 |
+
nbformat==5.10.4
|
| 154 |
+
nest-asyncio==1.6.0
|
| 155 |
+
networkx==3.5
|
| 156 |
+
ninja==1.11.1.4
|
| 157 |
+
notebook==7.4.3
|
| 158 |
+
notebook_shim==0.2.4
|
| 159 |
+
numba==0.59.1
|
| 160 |
+
numba-cuda @ file:///rapids/numba_cuda-0.4.0-py3-none-any.whl#sha256=5b95a5f42068771e37142944d44fe844fa87f1b2f89051461a4cb0af91bcc28a
|
| 161 |
+
numcodecs @ file:///rapids/numcodecs-0.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=da2230484e6102e5fa3cc1a5dd37ca1f92dfbd183d91662074d6f7574e3e8f53
|
| 162 |
+
numpy==1.26.4
|
| 163 |
+
nvdlfw_inspect @ git+https://github.com/NVIDIA/nvidia-dlfw-inspect.git@6b60eb0e675606fb2fbbfcb12667ecbec75eaf87
|
| 164 |
+
nvfuser==0.2.27a0+9bf5aca
|
| 165 |
+
nvidia-cuda-cccl-cu12==12.9.27
|
| 166 |
+
nvidia-cudnn-frontend @ file:///opt/pytorch/lightning-thunder/tmp/cudnn_frontend
|
| 167 |
+
nvidia-cutlass-dsl==4.4.2
|
| 168 |
+
nvidia-cutlass-dsl-libs-base==4.4.2
|
| 169 |
+
nvidia-dali-cuda120==1.50.0
|
| 170 |
+
nvidia-ml-py @ file:///rapids/nvidia_ml_py-12.575.51-py3-none-any.whl#sha256=eb8641800d98ce40a22f479873f34b482e214a7e80349c63be51c3919845446e
|
| 171 |
+
nvidia-modelopt==0.29.0
|
| 172 |
+
nvidia-modelopt-core==0.29.0
|
| 173 |
+
nvidia-nvcomp-cu12==4.2.0.14
|
| 174 |
+
nvidia-nvimgcodec-cu12==0.5.0.13
|
| 175 |
+
nvidia-nvjpeg-cu12==12.4.0.16
|
| 176 |
+
nvidia-nvjpeg2k-cu12==0.8.1.40
|
| 177 |
+
nvidia-nvshmem-cu12==3.7.2
|
| 178 |
+
nvidia-nvtiff-cu12==0.5.0.67
|
| 179 |
+
nvidia-resiliency-ext==0.4.0
|
| 180 |
+
nvtx @ file:///rapids/nvtx-0.2.11-cp312-cp312-linux_x86_64.whl#sha256=73521e37cba42b4db0b3cd68fc5eb814e55c500a61eeed6edcdf6746315ac53e
|
| 181 |
+
nx-cugraph @ file:///rapids/nx_cugraph-25.4.0-py3-none-any.whl#sha256=e625444486cc899a77bd48e7da76a2edd95c0006dd95fbf323745feae7f33111
|
| 182 |
+
onnx @ file:///opt/pytorch/pytorch/third_party/onnx
|
| 183 |
+
onnx-ir==1.0.0
|
| 184 |
+
onnxscript==0.7.2
|
| 185 |
+
opt_einsum==3.4.0
|
| 186 |
+
optree==0.16.0
|
| 187 |
+
overrides==7.7.0
|
| 188 |
+
packaging==23.2
|
| 189 |
+
pandas @ file:///rapids/pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319
|
| 190 |
+
pandocfilters==1.5.1
|
| 191 |
+
parso==0.8.4
|
| 192 |
+
partd @ file:///rapids/partd-1.4.2-py3-none-any.whl#sha256=978e4ac767ec4ba5b86c6eaa52e5a2a3bc748a2ca839e8cc798f1cc6ce6efb0f
|
| 193 |
+
pathspec==0.12.1
|
| 194 |
+
pexpect==4.9.0
|
| 195 |
+
pillow==11.2.1
|
| 196 |
+
platformdirs==4.3.8
|
| 197 |
+
pluggy==1.6.0
|
| 198 |
+
ply @ file:///rapids/ply-3.11-py2.py3-none-any.whl#sha256=096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce
|
| 199 |
+
polars @ file:///rapids/polars-1.25.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=f7fcbb4f476784384ccda48757fca4e8c2e2c5a0a3aef3717aaf56aee4e30e09
|
| 200 |
+
polygraphy==0.49.20
|
| 201 |
+
pooch==1.8.2
|
| 202 |
+
preshed==3.0.10
|
| 203 |
+
prometheus_client==0.22.1
|
| 204 |
+
prompt_toolkit==3.0.51
|
| 205 |
+
propcache @ file:///rapids/propcache-0.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=563f9d8c03ad645597b8d010ef4e9eab359faeb11a0a2ac9f7b4bc8c28ebef25
|
| 206 |
+
protobuf==4.24.4
|
| 207 |
+
psutil==7.0.0
|
| 208 |
+
ptyprocess==0.7.0
|
| 209 |
+
PuLP==3.2.1
|
| 210 |
+
pure_eval==0.2.3
|
| 211 |
+
pyarrow==25.0.1
|
| 212 |
+
pybind11==2.13.6
|
| 213 |
+
pybind11_global==2.13.6
|
| 214 |
+
pycocotools @ git+https://github.com/nvidia/cocoapi.git@08350a157b41f0755a536441b5abec89ab934c5b#subdirectory=PythonAPI
|
| 215 |
+
pycparser==2.22
|
| 216 |
+
pydantic==2.11.5
|
| 217 |
+
pydantic_core==2.33.2
|
| 218 |
+
Pygments==2.19.1
|
| 219 |
+
pylibcudf @ file:///rapids/pylibcudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=401b8bfd022d06f84e120779ca7c3d01a28b68762aaa8f7873b10d6b1975cf6a
|
| 220 |
+
pylibcugraph @ file:///rapids/pylibcugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=061c7389a88c815b54fd5d3a3584fb87bf693e5602f5be335260d0ba575ba557
|
| 221 |
+
pylibcugraphops @ file:///rapids/pylibcugraphops-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=a25ef4b22166c80718d5c7e5bddb755e81a2fe340fa5becd0ac36ec3edfb208a
|
| 222 |
+
pylibraft @ file:///rapids/pylibraft-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=f2ecd23fe51f9c2caa87bbbe85cb32fcaff4ecfb52ceb9a9991bc50f82782c2e
|
| 223 |
+
pylibwholegraph @ file:///rapids/pylibwholegraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b82913d1e38bda76d8c272fbd8cc9072d987209d3d47cc93a464a37c2925aca2
|
| 224 |
+
pynvjitlink @ file:///rapids/pynvjitlink-0.3.0-cp312-cp312-linux_x86_64.whl#sha256=066caf1314730643d3b48c9847bd65789defdd6a7c78993176eb3d8bbe3f4919
|
| 225 |
+
pynvml @ file:///rapids/pynvml-12.0.0-py3-none-any.whl#sha256=fdff84b62a27dbe98e08e1a647eb77342bef1aebe0878bcd15e99a83fcbecb9e
|
| 226 |
+
pyparsing==3.2.3
|
| 227 |
+
pytest==8.1.1
|
| 228 |
+
pytest-flakefinder==1.1.0
|
| 229 |
+
pytest-rerunfailures==15.1
|
| 230 |
+
pytest-shard==0.1.2
|
| 231 |
+
pytest-xdist==3.7.0
|
| 232 |
+
python-dateutil==2.9.0.post0
|
| 233 |
+
python-hostlist==2.2.1
|
| 234 |
+
python-json-logger==3.3.0
|
| 235 |
+
pytorch-triton @ file:///tmp/dist/pytorch_triton-3.3.0%2Bgit96316ce52.nvinternal-cp312-cp312-linux_x86_64.whl#sha256=6d6921772b9dfd0243c055e0d054de42a113e92cabcfdd1d3e80a5ad9c08cf7d
|
| 236 |
+
pytz @ file:///rapids/pytz-2023.4-py2.py3-none-any.whl#sha256=f90ef520d95e7c46951105338d918664ebfd6f1d995bd7d153127ce90efafa6a
|
| 237 |
+
PyYAML==6.0.2
|
| 238 |
+
pyzmq==26.4.0
|
| 239 |
+
raft-dask @ file:///rapids/raft_dask-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=36cf37ba6ce35454c4729c2cb597a360f22780eefb11d514a16a07137da5eb6c
|
| 240 |
+
rapids-dask-dependency @ file:///rapids/rapids_dask_dependency-25.4.0a0-py3-none-any.whl#sha256=484faca626e7760112a7d8ad3426f5bdd720b609006d8dca87c2607076a348bd
|
| 241 |
+
rapids-logger @ file:///rapids/rapids_logger-0.1.18-py3-none-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=3617b2d86291ac6cb957542895f20fc9737ace29773ca90337e6fb04c73bd120
|
| 242 |
+
referencing==0.36.2
|
| 243 |
+
regex==2024.11.6
|
| 244 |
+
requests==2.32.3
|
| 245 |
+
rfc3339-validator==0.1.4
|
| 246 |
+
rfc3986-validator==0.1.1
|
| 247 |
+
rich==14.0.0
|
| 248 |
+
rmm @ file:///rapids/rmm-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1824bcab3836296666980a997ddcb9c4bca24dfb873012cfa0a7ecea583ca712
|
| 249 |
+
rpds-py==0.25.1
|
| 250 |
+
safetensors==0.5.3
|
| 251 |
+
scikit-learn==1.6.1
|
| 252 |
+
scipy==1.15.3
|
| 253 |
+
Send2Trash==1.8.3
|
| 254 |
+
setuptools==78.1.1
|
| 255 |
+
shellingham==1.5.4
|
| 256 |
+
six==1.16.0
|
| 257 |
+
smart-open==7.1.0
|
| 258 |
+
sniffio==1.3.1
|
| 259 |
+
sortedcontainers==2.4.0
|
| 260 |
+
soundfile==0.13.1
|
| 261 |
+
soupsieve==2.7
|
| 262 |
+
soxr==0.5.0.post1
|
| 263 |
+
spacy==3.7.5
|
| 264 |
+
spacy-legacy==3.0.12
|
| 265 |
+
spacy-loggers==1.0.5
|
| 266 |
+
srsly==2.5.1
|
| 267 |
+
stack-data==0.6.3
|
| 268 |
+
sympy==1.14.0
|
| 269 |
+
tabulate==0.9.0
|
| 270 |
+
tbb==2021.13.1
|
| 271 |
+
tblib @ file:///rapids/tblib-3.1.0-py3-none-any.whl#sha256=670bb4582578134b3d81a84afa1b016128b429f3d48e6cbbaecc9d15675e984e
|
| 272 |
+
tensorboard==2.16.2
|
| 273 |
+
tensorboard-data-server==0.7.2
|
| 274 |
+
tensorrt @ file:///workspace/TensorRT-10.11.0.33/python/tensorrt-10.11.0.33-cp312-none-linux_x86_64.whl#sha256=c2f18f72758a3d54a71901495c8fda9e7094a8bcdd2a5b6b58967fbf05733b3a
|
| 275 |
+
terminado==0.18.1
|
| 276 |
+
thinc==8.2.5
|
| 277 |
+
threadpoolctl==3.6.0
|
| 278 |
+
thriftpy2 @ file:///rapids/thriftpy2-0.5.2-cp312-cp312-linux_x86_64.whl#sha256=c0c7d939bdc5ccf7739154959938445d5fdf72f1d6e12bd563947a36f67e2147
|
| 279 |
+
tinycss2==1.4.0
|
| 280 |
+
toolz @ file:///rapids/toolz-1.0.0-py3-none-any.whl#sha256=292c8f1c4e7516bf9086f8850935c799a874039c8bcf959d47b600e4c44a6236
|
| 281 |
+
torch @ file:///opt/transfer/torch-2.8.0a0%2B5228986c39.nv25.6-29946474-cp312-cp312-linux_x86_64.whl#sha256=7a3c9ff5288d8102a31296b3d62491f3f47c8820a5c9aaefc7dc0fe20e1e8ad5
|
| 282 |
+
torch_tensorrt @ file:///opt/pytorch/torch_tensorrt/dist/torch_tensorrt-2.8.0a0-cp312-cp312-linux_x86_64.whl#sha256=26f4d7a04e7e6e017a5b917b8b98b83720bb2bb0195f48d25f52d567d286bfe1
|
| 283 |
+
torchao @ file:///opt/pytorch/ao
|
| 284 |
+
torchprofile==0.0.4
|
| 285 |
+
torchvision @ file:///opt/transfer/torchvision-0.22.0a0%2B95f10a4e-29946474-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=746a3c674cc61680a86f4a14b374b21af1b5c21f629c42765b40c9494f6b234d
|
| 286 |
+
tornado==6.5.1
|
| 287 |
+
tqdm==4.67.1
|
| 288 |
+
traitlets==5.14.3
|
| 289 |
+
transformer_engine @ file:///pvcdata/aiinfra/b-aghosh/wheels/transformer_engine-2.10.0.dev0%2B10f6128d-cp312-cp312-linux_x86_64.whl#sha256=35cf5f4b83b2baad864e12e512bb422e0929409ac05610812fa521d4942d0189
|
| 290 |
+
treelite @ file:///rapids/treelite-4.4.1-py3-none-linux_x86_64.whl#sha256=174741724205540a1a5b37da81483aeff55d89548f42b3461cf07acac7ae96eb
|
| 291 |
+
typer==0.16.0
|
| 292 |
+
types-dataclasses==0.6.6
|
| 293 |
+
types-python-dateutil==2.9.0.20250516
|
| 294 |
+
typing-inspection==0.4.1
|
| 295 |
+
typing_extensions==4.14.0
|
| 296 |
+
tzdata @ file:///rapids/tzdata-2025.2-py2.py3-none-any.whl#sha256=1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8
|
| 297 |
+
ucx-py @ file:///rapids/ucx_py-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=4a452fd8d8c983283ada3d49086e3657dc2ee94fbad08efdd708d15ab6e0b73a
|
| 298 |
+
ucxx @ file:///rapids/ucxx-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=f1b4e55cb45c66fe88a2d8f9c2b87751165f3cfec21fbdc9c47c4ddff75e8a5f
|
| 299 |
+
-e git+https://github.com/Dots-Infra/UltraEP.git@94cab099b44fffa99a82fea99e7c12d89cf65e4f#egg=ultra_ep
|
| 300 |
+
uri-template==1.3.0
|
| 301 |
+
urllib3==2.0.7
|
| 302 |
+
wasabi==1.1.3
|
| 303 |
+
wcwidth==0.2.13
|
| 304 |
+
weasel==0.4.1
|
| 305 |
+
webcolors==24.11.1
|
| 306 |
+
webencodings==0.5.1
|
| 307 |
+
websocket-client==1.8.0
|
| 308 |
+
Werkzeug==3.1.3
|
| 309 |
+
wheel==0.45.1
|
| 310 |
+
wrapt==1.17.2
|
| 311 |
+
xdoctest==1.0.2
|
| 312 |
+
xgboost @ file:///rapids/xgboost-2.1.4-py3-none-manylinux_2_39_x86_64.whl#sha256=25eeca27f4048fc491940b3a0b4b5782ee48c33cb3aec059cef71f8da542930b
|
| 313 |
+
xxhash==4.0.1
|
| 314 |
+
yarl @ file:///rapids/yarl-1.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=bf099e2432131093cc611623e0b0bcc399b8cddd9a91eded8bfb50402ec35018
|
| 315 |
+
zarr @ file:///rapids/zarr-2.18.7-py3-none-any.whl#sha256=ac3dc4033e9ae4e9d7b5e27c97ea3eaf1003cc0a07f010bd83d5134bf8c4b223
|
| 316 |
+
zict @ file:///rapids/zict-3.0.0-py2.py3-none-any.whl#sha256=5796e36bd0e0cc8cf0fbc1ace6a68912611c1dbd74750a3f3026b9b9d6a327ae
|
| 317 |
+
zipp==3.22.0
|
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/provenance.txt
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
==============================================================
|
| 2 |
+
PROVENANCE: ultraep__tierb__gbs8
|
| 3 |
+
==============================================================
|
| 4 |
+
captured_at : 2026-09-21T07:47:12+00:00
|
| 5 |
+
hostname : sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0
|
| 6 |
+
kernel : 6.8.0-1043-nvidia
|
| 7 |
+
cwd : /workspace/blackwell_run_scripts
|
| 8 |
+
command : run_ladder.sh --phase tierb --gbs 8 --out /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb --nsys
|
| 9 |
+
|
| 10 |
+
---- GPU / driver --------------------------------------------
|
| 11 |
+
0, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
|
| 12 |
+
1, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
|
| 13 |
+
2, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
|
| 14 |
+
3, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
|
| 15 |
+
4, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
|
| 16 |
+
5, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
|
| 17 |
+
6, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
|
| 18 |
+
7, NVIDIA B200, 10.0, 183359 MiB, 580.95.05
|
| 19 |
+
persistence/clocks:
|
| 20 |
+
0, 120 MHz, 1965 MHz, 1000.00 W, 29
|
| 21 |
+
1, 1965 MHz, 1965 MHz, 1000.00 W, 35
|
| 22 |
+
2, 120 MHz, 1965 MHz, 1000.00 W, 35
|
| 23 |
+
3, 262 MHz, 1965 MHz, 1000.00 W, 30
|
| 24 |
+
4, 1965 MHz, 1965 MHz, 1000.00 W, 30
|
| 25 |
+
5, 120 MHz, 1965 MHz, 1000.00 W, 34
|
| 26 |
+
6, 165 MHz, 1965 MHz, 1000.00 W, 34
|
| 27 |
+
7, 120 MHz, 1965 MHz, 1000.00 W, 30
|
| 28 |
+
|
| 29 |
+
NVLink topology:
|
| 30 |
+
[4mGPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 CPU Affinity NUMA Affinity GPU NUMA ID[0m
|
| 31 |
+
GPU0 X NV18 NV18 NV18 NV18 NV18 NV18 NV18 PIX NODE SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A
|
| 32 |
+
GPU1 NV18 X NV18 NV18 NV18 NV18 NV18 NV18 NODE PIX SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A
|
| 33 |
+
GPU2 NV18 NV18 X NV18 NV18 NV18 NV18 NV18 SYS SYS PIX NODE SYS SYS SYS SYS SYS 28-55 1 N/A
|
| 34 |
+
GPU3 NV18 NV18 NV18 X NV18 NV18 NV18 NV18 SYS SYS NODE PIX SYS SYS SYS SYS SYS 28-55 1 N/A
|
| 35 |
+
GPU4 NV18 NV18 NV18 NV18 X NV18 NV18 NV18 SYS SYS SYS SYS PIX NODE SYS SYS SYS 56-83 2 N/A
|
| 36 |
+
GPU5 NV18 NV18 NV18 NV18 NV18 X NV18 NV18 SYS SYS SYS SYS NODE PIX SYS SYS SYS 56-83 2 N/A
|
| 37 |
+
GPU6 NV18 NV18 NV18 NV18 NV18 NV18 X NV18 SYS SYS SYS SYS SYS SYS PIX NODE SYS 84-111 3 N/A
|
| 38 |
+
GPU7 NV18 NV18 NV18 NV18 NV18 NV18 NV18 X SYS SYS SYS SYS SYS SYS NODE PIX SYS 84-111 3 N/A
|
| 39 |
+
NIC0 PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS SYS NODE
|
| 40 |
+
NIC1 NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS SYS NODE
|
| 41 |
+
NIC2 SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS
|
| 42 |
+
NIC3 SYS SYS NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS
|
| 43 |
+
NIC4 SYS SYS SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS
|
| 44 |
+
unavailable
|
| 45 |
+
|
| 46 |
+
---- toolchain -----------------------------------------------
|
| 47 |
+
nvcc : release 12.9, V12.9.86
|
| 48 |
+
gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0
|
| 49 |
+
python : 3.12.3
|
| 50 |
+
torch : 2.8.0a0+5228986c39.nv25.06 cuda=12.9 nccl=2.27.3
|
| 51 |
+
torch arch : ['sm_75', 'sm_80', 'sm_86', 'sm_90', 'sm_100', 'sm_120', 'compute_120']
|
| 52 |
+
transformer_engine: 2.10.0.dev0+10f6128d [/usr/local/lib/python3.12/dist-packages/transformer_engine/__init__.py]
|
| 53 |
+
deep_ep : 1.2.1+34152ae [/workspace/DeepEP/deep_ep/__init__.py]
|
| 54 |
+
ultra_ep : 1.0.0 [/workspace/UltraEP/ultra_ep/__init__.py]
|
| 55 |
+
moonep : 0.0.1 [/workspace/MoonEP/moonep/__init__.py]
|
| 56 |
+
megatron : 0.14.0rc5 [None]
|
| 57 |
+
flash_attn : 2.7.4.post1 [/usr/local/lib/python3.12/dist-packages/flash_attn/__init__.py]
|
| 58 |
+
numpy : 1.26.4 [/usr/local/lib/python3.12/dist-packages/numpy/__init__.py]
|
| 59 |
+
huggingface_hub: 1.32.0 [/usr/local/lib/python3.12/dist-packages/huggingface_hub/__init__.py]
|
| 60 |
+
|
| 61 |
+
---- repositories --------------------------------------------
|
| 62 |
+
Megatron-MoE-Echo bracing-regions-through-kernels 9a23679390d7
|
| 63 |
+
Megatron-UltraEP ultraep-bracing-regions-through-kernels 03d57ef7d476
|
| 64 |
+
UltraEP main 94cab099b44f
|
| 65 |
+
DeepEP HEAD 34152ae28f80 (+3 untracked: build output)
|
| 66 |
+
MoonEP megatron-ddp-symmetric-params d8bc6667fad7
|
| 67 |
+
blackwell_run_scripts granularity-ultraep 05dd2cbfb1eb
|
| 68 |
+
|
| 69 |
+
---- environment ---------------------------------------------
|
| 70 |
+
CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0
|
| 71 |
+
CUDA_DRIVER_VERSION=575.57.08
|
| 72 |
+
CUDA_HOME=/usr/local/cuda
|
| 73 |
+
CUDA_MODULE_LOADING=LAZY
|
| 74 |
+
CUDA_VERSION=12.9.1.010
|
| 75 |
+
HF_REMOTE_ROOT=granularity_next_b200/tierb_models/attempts/KIMI_K2_L2
|
| 76 |
+
NCCL_VERSION=2.27.3
|
| 77 |
+
NVSHMEM_VERSION=3.2.5
|
| 78 |
+
PYTORCH_BUILD_NUMBER=0
|
| 79 |
+
PYTORCH_BUILD_VERSION=2.8.0a0+5228986
|
| 80 |
+
PYTORCH_HOME=/opt/pytorch/pytorch
|
| 81 |
+
PYTORCH_VERSION=2.8.0a0+5228986
|
| 82 |
+
TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1
|
| 83 |
+
TORCH_CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0+PTX
|
| 84 |
+
TORCH_NCCL_USE_COMM_NONBLOCKING=0
|
| 85 |
+
|
| 86 |
+
---- shared library resolution -------------------------------
|
| 87 |
+
libnvshmem_host.so: not in ldconfig
|
| 88 |
+
libnccl.so: /lib/x86_64-linux-gnu/libnccl.so.2
|
| 89 |
+
lrwxrwxrwx 1 root root 20 Sep 21 04:57 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so -> libnvshmem_host.so.3
|
| 90 |
+
-rw-r--r-- 1 root root 179045744 Sep 21 04:54 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so.3
|
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep1/mem_samples.csv
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
0, 0
|
| 2 |
+
1, 0
|
| 3 |
+
2, 0
|
| 4 |
+
3, 0
|
| 5 |
+
4, 0
|
| 6 |
+
5, 0
|
| 7 |
+
6, 0
|
| 8 |
+
7, 0
|
| 9 |
+
0, 0
|
| 10 |
+
1, 0
|
| 11 |
+
2, 0
|
| 12 |
+
3, 0
|
| 13 |
+
4, 0
|
| 14 |
+
5, 0
|
| 15 |
+
6, 0
|
| 16 |
+
7, 0
|
| 17 |
+
0, 1
|
| 18 |
+
1, 1
|
| 19 |
+
2, 1
|
| 20 |
+
3, 1
|
| 21 |
+
4, 1
|
| 22 |
+
5, 1
|
| 23 |
+
6, 1
|
| 24 |
+
7, 1
|
| 25 |
+
0, 4
|
| 26 |
+
1, 4
|
| 27 |
+
2, 4
|
| 28 |
+
3, 4
|
| 29 |
+
4, 1
|
| 30 |
+
5, 1
|
| 31 |
+
6, 1
|
| 32 |
+
7, 1
|
| 33 |
+
0, 4
|
| 34 |
+
1, 4
|
| 35 |
+
2, 4
|
| 36 |
+
3, 4
|
| 37 |
+
4, 4
|
| 38 |
+
5, 4
|
| 39 |
+
6, 4
|
| 40 |
+
7, 4
|
| 41 |
+
0, 4
|
| 42 |
+
1, 629
|
| 43 |
+
2, 629
|
| 44 |
+
3, 629
|
| 45 |
+
4, 629
|
| 46 |
+
5, 629
|
| 47 |
+
6, 629
|
| 48 |
+
7, 629
|
| 49 |
+
0, 887
|
| 50 |
+
1, 885
|
| 51 |
+
2, 915
|
| 52 |
+
3, 881
|
| 53 |
+
4, 885
|
| 54 |
+
5, 905
|
| 55 |
+
6, 909
|
| 56 |
+
7, 903
|
| 57 |
+
0, 1927
|
| 58 |
+
1, 1927
|
| 59 |
+
2, 1927
|
| 60 |
+
3, 1927
|
| 61 |
+
4, 1927
|
| 62 |
+
5, 1927
|
| 63 |
+
6, 1927
|
| 64 |
+
7, 1927
|
| 65 |
+
0, 3783
|
| 66 |
+
1, 3789
|
| 67 |
+
2, 3803
|
| 68 |
+
3, 3815
|
| 69 |
+
4, 3807
|
| 70 |
+
5, 3803
|
| 71 |
+
6, 3811
|
| 72 |
+
7, 3809
|
| 73 |
+
0, 50559
|
| 74 |
+
1, 50559
|
| 75 |
+
2, 50559
|
| 76 |
+
3, 50559
|
| 77 |
+
4, 50559
|
| 78 |
+
5, 50559
|
| 79 |
+
6, 50559
|
| 80 |
+
7, 50559
|
| 81 |
+
0, 55555
|
| 82 |
+
1, 55555
|
| 83 |
+
2, 55555
|
| 84 |
+
3, 55555
|
| 85 |
+
4, 55555
|
| 86 |
+
5, 55555
|
| 87 |
+
6, 55555
|
| 88 |
+
7, 55555
|
| 89 |
+
0, 60691
|
| 90 |
+
1, 60693
|
| 91 |
+
2, 60691
|
| 92 |
+
3, 60691
|
| 93 |
+
4, 60691
|
| 94 |
+
5, 60691
|
| 95 |
+
6, 60691
|
| 96 |
+
7, 60691
|
| 97 |
+
0, 61267
|
| 98 |
+
1, 61269
|
| 99 |
+
2, 61267
|
| 100 |
+
3, 61267
|
| 101 |
+
4, 61267
|
| 102 |
+
5, 61267
|
| 103 |
+
6, 61267
|
| 104 |
+
7, 61267
|
| 105 |
+
0, 61267
|
| 106 |
+
1, 61269
|
| 107 |
+
2, 61267
|
| 108 |
+
3, 61267
|
| 109 |
+
4, 61267
|
| 110 |
+
5, 61267
|
| 111 |
+
6, 61267
|
| 112 |
+
7, 61267
|
| 113 |
+
0, 64465
|
| 114 |
+
1, 64467
|
| 115 |
+
2, 64465
|
| 116 |
+
3, 64465
|
| 117 |
+
4, 64465
|
| 118 |
+
5, 64465
|
| 119 |
+
6, 64465
|
| 120 |
+
7, 64465
|
| 121 |
+
0, 64465
|
| 122 |
+
1, 64467
|
| 123 |
+
2, 64465
|
| 124 |
+
3, 64465
|
| 125 |
+
4, 64465
|
| 126 |
+
5, 64465
|
| 127 |
+
6, 64465
|
| 128 |
+
7, 64465
|
| 129 |
+
0, 65963
|
| 130 |
+
1, 65965
|
| 131 |
+
2, 65963
|
| 132 |
+
3, 65963
|
| 133 |
+
4, 65963
|
| 134 |
+
5, 65963
|
| 135 |
+
6, 65963
|
| 136 |
+
7, 65963
|
| 137 |
+
0, 65963
|
| 138 |
+
1, 65965
|
| 139 |
+
2, 65963
|
| 140 |
+
3, 65963
|
| 141 |
+
4, 65963
|
| 142 |
+
5, 65963
|
| 143 |
+
6, 65963
|
| 144 |
+
7, 65963
|
| 145 |
+
0, 65969
|
| 146 |
+
1, 65969
|
| 147 |
+
2, 65969
|
| 148 |
+
3, 65969
|
| 149 |
+
4, 65969
|
| 150 |
+
5, 65969
|
| 151 |
+
6, 65969
|
| 152 |
+
7, 65969
|
| 153 |
+
0, 68251
|
| 154 |
+
1, 68233
|
| 155 |
+
2, 68277
|
| 156 |
+
3, 68259
|
| 157 |
+
4, 68265
|
| 158 |
+
5, 68251
|
| 159 |
+
6, 68297
|
| 160 |
+
7, 68303
|
| 161 |
+
0, 69055
|
| 162 |
+
1, 69055
|
| 163 |
+
2, 69055
|
| 164 |
+
3, 69055
|
| 165 |
+
4, 69055
|
| 166 |
+
5, 69055
|
| 167 |
+
6, 69055
|
| 168 |
+
7, 69055
|
| 169 |
+
0, 75415
|
| 170 |
+
1, 75387
|
| 171 |
+
2, 75419
|
| 172 |
+
3, 75397
|
| 173 |
+
4, 75471
|
| 174 |
+
5, 75487
|
| 175 |
+
6, 75405
|
| 176 |
+
7, 75459
|
| 177 |
+
0, 88919
|
| 178 |
+
1, 89733
|
| 179 |
+
2, 86735
|
| 180 |
+
3, 89731
|
| 181 |
+
4, 85699
|
| 182 |
+
5, 88975
|
| 183 |
+
6, 89745
|
| 184 |
+
7, 85223
|
| 185 |
+
0, 90291
|
| 186 |
+
1, 90297
|
| 187 |
+
2, 90291
|
| 188 |
+
3, 90347
|
| 189 |
+
4, 90347
|
| 190 |
+
5, 90347
|
| 191 |
+
6, 90375
|
| 192 |
+
7, 90347
|
| 193 |
+
0, 13828
|
| 194 |
+
1, 13828
|
| 195 |
+
2, 14148
|
| 196 |
+
3, 89125
|
| 197 |
+
4, 15428
|
| 198 |
+
5, 13828
|
| 199 |
+
6, 13828
|
| 200 |
+
7, 13828
|
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep1/run.env
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=ultraep
|
| 2 |
+
config=KIMI_K2_L2
|
| 3 |
+
family=tierB
|
| 4 |
+
source_model=Kimi-K2
|
| 5 |
+
standard_num_layers=61
|
| 6 |
+
base_rung=
|
| 7 |
+
repo=/workspace/Megatron-UltraEP
|
| 8 |
+
branch=ultraep-bracing-regions-through-kernels
|
| 9 |
+
commit=03d57ef7
|
| 10 |
+
num_layers=2
|
| 11 |
+
hidden=7168
|
| 12 |
+
ffn_hidden=18432
|
| 13 |
+
moe_ffn_hidden=2048
|
| 14 |
+
num_experts=384
|
| 15 |
+
topk=8
|
| 16 |
+
experts_per_rank=48
|
| 17 |
+
expert_params=44040192
|
| 18 |
+
heads=64
|
| 19 |
+
kv_groups=64
|
| 20 |
+
kv_channels=64
|
| 21 |
+
qk_layernorm=1
|
| 22 |
+
shared_ffn=2048
|
| 23 |
+
shared_expert_overlap=0
|
| 24 |
+
moe_token_dispatcher=flex
|
| 25 |
+
mla=1
|
| 26 |
+
q_lora_rank=1536
|
| 27 |
+
kv_lora_rank=512
|
| 28 |
+
qk_head_dim=128
|
| 29 |
+
qk_pos_emb_head_dim=64
|
| 30 |
+
v_head_dim=128
|
| 31 |
+
dense_first_layers=1
|
| 32 |
+
moe_layers=1
|
| 33 |
+
rotary_base=50000
|
| 34 |
+
rotary_percent=1.0
|
| 35 |
+
rotary_scaling=32
|
| 36 |
+
mscale=1.0
|
| 37 |
+
mscale_all_dim=1.0
|
| 38 |
+
qkv_bias=0
|
| 39 |
+
router_balancing=seq_aux_loss
|
| 40 |
+
aux_loss_coeff=0
|
| 41 |
+
router_pre_softmax=0
|
| 42 |
+
router_fusion=0
|
| 43 |
+
router_dtype=fp32
|
| 44 |
+
router_groups=1
|
| 45 |
+
router_group_topk=1
|
| 46 |
+
router_score_function=sigmoid
|
| 47 |
+
router_expert_bias=1
|
| 48 |
+
router_bias_update_rate=1e-3
|
| 49 |
+
router_topk_scaling_factor=2.827
|
| 50 |
+
shared_expert_gate=0
|
| 51 |
+
apply_layernorm_1p=0
|
| 52 |
+
attention_output_gate=0
|
| 53 |
+
linear_attention_type=
|
| 54 |
+
linear_attention_freq=
|
| 55 |
+
linear_conv_kernel_dim=
|
| 56 |
+
linear_key_head_dim=
|
| 57 |
+
linear_value_head_dim=
|
| 58 |
+
linear_num_key_heads=
|
| 59 |
+
linear_num_value_heads=
|
| 60 |
+
mtp_num_layers=0
|
| 61 |
+
mtp_loss_scaling_factor=0.1
|
| 62 |
+
no_weight_decay_cond_type=
|
| 63 |
+
max_position_embeddings=4096
|
| 64 |
+
init_method_std=0.02
|
| 65 |
+
device_initiated_grouped_gemm=1
|
| 66 |
+
norm_epsilon=1e-6
|
| 67 |
+
seq_length=4096
|
| 68 |
+
micro_batch_size=1
|
| 69 |
+
global_batch_size=8
|
| 70 |
+
num_microbatches=1
|
| 71 |
+
assignments_per_rank=32768
|
| 72 |
+
mean_tokens_per_expert=682
|
| 73 |
+
slot_policy=matched
|
| 74 |
+
slots_per_rank=48
|
| 75 |
+
capacity_mode=static
|
| 76 |
+
capacity=1.17578125
|
| 77 |
+
cuda_graph=1
|
| 78 |
+
probe_mode=0
|
| 79 |
+
async_grad_reduce=0
|
| 80 |
+
branch_role=timing
|
| 81 |
+
memory_dump_mode=none
|
| 82 |
+
memory_snapshot_path=
|
| 83 |
+
cuda_graph_warmup_steps=3
|
| 84 |
+
recompute=none
|
| 85 |
+
force_balance=0
|
| 86 |
+
train_iters=12
|
| 87 |
+
ep_size=8
|
| 88 |
+
echo_dispatch_overlap=1
|
| 89 |
+
echo_dispatch_recompute=0
|
| 90 |
+
ultraep_quota_min_tokens=85
|
| 91 |
+
ultraep_weight_sync_join=
|
| 92 |
+
profile_enable=0
|
| 93 |
+
kernel_trace=0
|
| 94 |
+
fp8_mode=mxfp8
|
| 95 |
+
fp8_recipe=mxfp8
|
| 96 |
+
training_seed=1234
|
| 97 |
+
learning_rate=1e-4
|
| 98 |
+
min_learning_rate=1.0e-5
|
| 99 |
+
lr_warmup_iters=5
|
| 100 |
+
state_dump_enable=0
|
| 101 |
+
state_dump_dir=
|
| 102 |
+
moonep_assert_no_pending_reduce=0
|
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep1/train.log
ADDED
|
@@ -0,0 +1,1073 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[07:47:26] WARN: ultraep-bracing-regions-through-kernels lacks --moe-router-fusion; KIMI_K2_L2 runs unfused (timing only, routing identical)
|
| 2 |
+
==============================================================
|
| 3 |
+
KIMI_K2_L2 / ultraep (slots/rank=48 policy=matched)
|
| 4 |
+
E=384 K=8 H'=2048 m=48
|
| 5 |
+
L=2 seq=4096 gbs=8 (1 microbatches)
|
| 6 |
+
cap_mode=static cap=1.17578125 cg=1 recompute=none
|
| 7 |
+
memory_dump=none snapshots=/workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep1/memory_snapshots/snapshot.pickle
|
| 8 |
+
repo=/workspace/Megatron-UltraEP @ 03d57ef7
|
| 9 |
+
==============================================================
|
| 10 |
+
W0921 07:47:28.271000 519691 torch/distributed/run.py:766]
|
| 11 |
+
W0921 07:47:28.271000 519691 torch/distributed/run.py:766] *****************************************
|
| 12 |
+
W0921 07:47:28.271000 519691 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 13 |
+
W0921 07:47:28.271000 519691 torch/distributed/run.py:766] *****************************************
|
| 14 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 15 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 16 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 17 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 18 |
+
using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1
|
| 19 |
+
WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer
|
| 20 |
+
Number of virtual stages per pipeline stage: None
|
| 21 |
+
accumulate and all-reduce gradients in fp32 for bfloat16 data type.
|
| 22 |
+
using torch.bfloat16 for parameters ...
|
| 23 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 24 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 25 |
+
------------------------ arguments ------------------------
|
| 26 |
+
account_for_embedding_in_pipeline_split ......... False
|
| 27 |
+
account_for_loss_in_pipeline_split .............. False
|
| 28 |
+
accumulate_allreduce_grads_in_fp32 .............. True
|
| 29 |
+
adam_beta1 ...................................... 0.9
|
| 30 |
+
adam_beta2 ...................................... 0.999
|
| 31 |
+
adam_eps ........................................ 1e-08
|
| 32 |
+
add_bias_linear ................................. False
|
| 33 |
+
add_position_embedding .......................... False
|
| 34 |
+
add_qkv_bias .................................... False
|
| 35 |
+
adlr_autoresume ................................. False
|
| 36 |
+
adlr_autoresume_interval ........................ 1000
|
| 37 |
+
align_grad_reduce ............................... True
|
| 38 |
+
align_param_gather .............................. False
|
| 39 |
+
app_tag_run_name ................................ None
|
| 40 |
+
app_tag_run_version ............................. 0.0.0
|
| 41 |
+
apply_layernorm_1p .............................. False
|
| 42 |
+
apply_query_key_layer_scaling ................... False
|
| 43 |
+
apply_residual_connection_post_layernorm ........ False
|
| 44 |
+
apply_rope_fusion ............................... False
|
| 45 |
+
async_save ...................................... None
|
| 46 |
+
async_tensor_model_parallel_allreduce ........... True
|
| 47 |
+
attention_backend ............................... AttnBackend.auto
|
| 48 |
+
attention_dropout ............................... 0.0
|
| 49 |
+
attention_softmax_in_fp32 ....................... False
|
| 50 |
+
auto_detect_ckpt_format ......................... False
|
| 51 |
+
barrier_with_L1_time ............................ True
|
| 52 |
+
bert_binary_head ................................ True
|
| 53 |
+
bert_embedder_type .............................. megatron
|
| 54 |
+
bert_load ....................................... None
|
| 55 |
+
bf16 ............................................ True
|
| 56 |
+
bias_dropout_fusion ............................. True
|
| 57 |
+
bias_gelu_fusion ................................ False
|
| 58 |
+
bias_swiglu_fusion .............................. True
|
| 59 |
+
biencoder_projection_dim ........................ 0
|
| 60 |
+
biencoder_shared_query_context_model ............ False
|
| 61 |
+
block_data_path ................................. None
|
| 62 |
+
calc_ft_timeouts ................................ False
|
| 63 |
+
calculate_per_token_loss ........................ False
|
| 64 |
+
check_for_large_grads ........................... False
|
| 65 |
+
check_for_nan_in_loss_and_grad .................. False
|
| 66 |
+
check_for_spiky_loss ............................ False
|
| 67 |
+
check_weight_hash_across_dp_replicas_interval ... None
|
| 68 |
+
ckpt_assume_constant_structure .................. False
|
| 69 |
+
ckpt_convert_format ............................. None
|
| 70 |
+
ckpt_convert_save ............................... None
|
| 71 |
+
ckpt_convert_update_legacy_dist_opt_format ...... False
|
| 72 |
+
ckpt_format ..................................... torch_dist
|
| 73 |
+
ckpt_fully_parallel_load ........................ False
|
| 74 |
+
ckpt_fully_parallel_save ........................ True
|
| 75 |
+
ckpt_fully_parallel_save_deprecated ............. False
|
| 76 |
+
ckpt_step ....................................... None
|
| 77 |
+
classes_fraction ................................ 1.0
|
| 78 |
+
clip_grad ....................................... 1.0
|
| 79 |
+
clone_scatter_output_in_embedding ............... True
|
| 80 |
+
config_logger_dir ...............................
|
| 81 |
+
consumed_train_samples .......................... 0
|
| 82 |
+
consumed_valid_samples .......................... 0
|
| 83 |
+
context_parallel_size ........................... 1
|
| 84 |
+
cp_comm_type .................................... ['p2p']
|
| 85 |
+
create_attention_mask_in_dataloader ............. True
|
| 86 |
+
cross_entropy_fusion_impl ....................... native
|
| 87 |
+
cross_entropy_loss_fusion ....................... False
|
| 88 |
+
cuda_graph_scope ................................ full_iteration
|
| 89 |
+
cuda_graph_warmup_steps ......................... 3
|
| 90 |
+
data_args_path .................................. None
|
| 91 |
+
data_cache_path ................................. None
|
| 92 |
+
data_parallel_random_init ....................... False
|
| 93 |
+
data_parallel_sharding_strategy ................. no_shard
|
| 94 |
+
data_parallel_size .............................. 8
|
| 95 |
+
data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document']
|
| 96 |
+
data_per_class_fraction ......................... 1.0
|
| 97 |
+
data_sharding ................................... True
|
| 98 |
+
dataloader_type ................................. single
|
| 99 |
+
ddp_average_in_collective ....................... False
|
| 100 |
+
ddp_bucket_size ................................. None
|
| 101 |
+
ddp_num_buckets ................................. None
|
| 102 |
+
ddp_pad_buckets_for_high_nccl_busbw ............. False
|
| 103 |
+
decoder_first_pipeline_num_layers ............... None
|
| 104 |
+
decoder_last_pipeline_num_layers ................ None
|
| 105 |
+
decoder_num_layers .............................. None
|
| 106 |
+
decoder_seq_length .............................. None
|
| 107 |
+
decoupled_lr .................................... None
|
| 108 |
+
decoupled_min_lr ................................ None
|
| 109 |
+
decrease_batch_size_if_needed ................... False
|
| 110 |
+
defer_embedding_wgrad_compute ................... False
|
| 111 |
+
delay_wgrad_compute ............................. False
|
| 112 |
+
deprecated_use_mcore_models ..................... True
|
| 113 |
+
deterministic_mode .............................. False
|
| 114 |
+
dino_bottleneck_size ............................ 256
|
| 115 |
+
dino_freeze_last_layer .......................... 1
|
| 116 |
+
dino_head_hidden_size ........................... 2048
|
| 117 |
+
dino_local_crops_number ......................... 10
|
| 118 |
+
dino_local_img_size ............................. 96
|
| 119 |
+
dino_norm_last_layer ............................ False
|
| 120 |
+
dino_teacher_temp ............................... 0.07
|
| 121 |
+
dino_warmup_teacher_temp ........................ 0.04
|
| 122 |
+
dino_warmup_teacher_temp_epochs ................. 30
|
| 123 |
+
disable_bf16_reduced_precision_matmul ........... False
|
| 124 |
+
disable_mamba_mem_eff_path ...................... False
|
| 125 |
+
disable_straggler_on_startup .................... False
|
| 126 |
+
dist_ckpt_format_deprecated ..................... None
|
| 127 |
+
dist_ckpt_strictness ............................ assume_ok_unexpected
|
| 128 |
+
distribute_saved_activations .................... False
|
| 129 |
+
distributed_backend ............................. nccl
|
| 130 |
+
distributed_timeout_minutes ..................... 10
|
| 131 |
+
embedding_init_method_std ....................... None
|
| 132 |
+
embedding_path .................................. None
|
| 133 |
+
empty_unused_memory_level ....................... 0
|
| 134 |
+
enable_cuda_graph ............................... True
|
| 135 |
+
enable_experimental ............................. False
|
| 136 |
+
enable_ft_package ............................... False
|
| 137 |
+
enable_gloo_process_groups ...................... True
|
| 138 |
+
enable_msc ...................................... True
|
| 139 |
+
enable_one_logger ............................... True
|
| 140 |
+
encoder_num_layers .............................. 2
|
| 141 |
+
encoder_seq_length .............................. 4096
|
| 142 |
+
end_weight_decay ................................ 0.1
|
| 143 |
+
eod_mask_loss ................................... False
|
| 144 |
+
error_injection_rate ............................ 0
|
| 145 |
+
error_injection_type ............................ transient_error
|
| 146 |
+
eval_interval ................................... 1000000
|
| 147 |
+
eval_iters ...................................... 0
|
| 148 |
+
evidence_data_path .............................. None
|
| 149 |
+
exit_duration_in_mins ........................... None
|
| 150 |
+
exit_interval ................................... None
|
| 151 |
+
exit_on_missing_checkpoint ...................... False
|
| 152 |
+
exit_signal_handler ............................. False
|
| 153 |
+
exp_avg_dtype ................................... torch.float32
|
| 154 |
+
exp_avg_sq_dtype ................................ torch.float32
|
| 155 |
+
expert_model_parallel_size ...................... 8
|
| 156 |
+
expert_tensor_parallel_size ..................... 1
|
| 157 |
+
export_force_local_attention .................... False
|
| 158 |
+
export_kd_cfg ................................... None
|
| 159 |
+
export_kd_teacher_ckpt_format ................... None
|
| 160 |
+
export_kd_teacher_load .......................... None
|
| 161 |
+
export_kv_cache_quant ........................... False
|
| 162 |
+
export_legacy_megatron .......................... False
|
| 163 |
+
export_model_type ............................... GPTModel
|
| 164 |
+
export_moe_apply_probs_on_input ................. False
|
| 165 |
+
export_qk_l2_norm ............................... False
|
| 166 |
+
export_quant_cfg ................................ None
|
| 167 |
+
export_real_quant_cfg ........................... None
|
| 168 |
+
export_te_mcore_model ........................... False
|
| 169 |
+
external_cuda_graph ............................. False
|
| 170 |
+
ffn_hidden_size ................................. 18432
|
| 171 |
+
finetune ........................................ False
|
| 172 |
+
finetune_data_split ............................. train
|
| 173 |
+
finetune_hf_dataset ............................. None
|
| 174 |
+
first_last_layers_bf16 .......................... False
|
| 175 |
+
flash_decode .................................... False
|
| 176 |
+
fp16 ............................................ False
|
| 177 |
+
fp16_lm_cross_entropy ........................... False
|
| 178 |
+
fp32_residual_connection ........................ False
|
| 179 |
+
fp8 ............................................. e4m3
|
| 180 |
+
fp8_amax_compute_algo ........................... most_recent
|
| 181 |
+
fp8_amax_history_len ............................ 1
|
| 182 |
+
fp8_interval .................................... 1
|
| 183 |
+
fp8_margin ...................................... 0
|
| 184 |
+
fp8_param_gather ................................ False
|
| 185 |
+
fp8_recipe ...................................... mxfp8
|
| 186 |
+
fp8_wgrad ....................................... True
|
| 187 |
+
fsdp_double_buffer .............................. False
|
| 188 |
+
global_batch_size ............................... 8
|
| 189 |
+
grad_reduce_in_bf16 ............................. False
|
| 190 |
+
gradient_accumulation_fusion .................... True
|
| 191 |
+
gradient_reduce_div_fusion ...................... True
|
| 192 |
+
group_query_attention ........................... False
|
| 193 |
+
head_lr_mult .................................... 1.0
|
| 194 |
+
heterogeneous_layers_config_encoded_json ........ None
|
| 195 |
+
heterogeneous_layers_config_path ................ None
|
| 196 |
+
hidden_dropout .................................. 0.0
|
| 197 |
+
hidden_size ..................................... 7168
|
| 198 |
+
hierarchical_context_parallel_sizes ............. None
|
| 199 |
+
high_priority_stream_groups ..................... []
|
| 200 |
+
hybrid_attention_ratio .......................... 0.0
|
| 201 |
+
hybrid_mlp_ratio ................................ 0.0
|
| 202 |
+
hybrid_override_pattern ......................... None
|
| 203 |
+
hysteresis ...................................... 2
|
| 204 |
+
ict_head_size ................................... None
|
| 205 |
+
ict_load ........................................ None
|
| 206 |
+
img_h ........................................... 224
|
| 207 |
+
img_w ........................................... 224
|
| 208 |
+
indexer_batch_size .............................. 128
|
| 209 |
+
indexer_log_interval ............................ 1000
|
| 210 |
+
inference_batch_times_seqlen_threshold .......... -1
|
| 211 |
+
inference_dynamic_batching ...................... False
|
| 212 |
+
inference_dynamic_batching_buffer_guaranteed_fraction 0.2
|
| 213 |
+
inference_dynamic_batching_buffer_overflow_factor None
|
| 214 |
+
inference_dynamic_batching_buffer_size_gb ....... 40.0
|
| 215 |
+
inference_dynamic_batching_chunk_size ........... 256
|
| 216 |
+
inference_dynamic_batching_max_requests_override None
|
| 217 |
+
inference_dynamic_batching_max_tokens_override .. None
|
| 218 |
+
inference_dynamic_batching_num_cuda_graphs ...... 16
|
| 219 |
+
inference_max_batch_size ........................ 8
|
| 220 |
+
inference_max_seq_length ........................ 2560
|
| 221 |
+
inference_rng_tracker ........................... False
|
| 222 |
+
init_method_std ................................. 0.02
|
| 223 |
+
init_method_xavier_uniform ...................... False
|
| 224 |
+
init_model_with_meta_device ..................... False
|
| 225 |
+
initial_loss_scale .............................. 4294967296
|
| 226 |
+
inprocess_active_world_size ..................... 8
|
| 227 |
+
inprocess_barrier_timeout ....................... 120
|
| 228 |
+
inprocess_completion_timeout .................... 120
|
| 229 |
+
inprocess_empty_cuda_cache ...................... False
|
| 230 |
+
inprocess_granularity ........................... node
|
| 231 |
+
inprocess_hard_timeout .......................... 90
|
| 232 |
+
inprocess_heartbeat_interval .................... 30
|
| 233 |
+
inprocess_heartbeat_timeout ..................... 60
|
| 234 |
+
inprocess_last_call_wait ........................ 1
|
| 235 |
+
inprocess_max_iterations ........................ None
|
| 236 |
+
inprocess_monitor_process_interval .............. 1.0
|
| 237 |
+
inprocess_monitor_thread_interval ............... 1.0
|
| 238 |
+
inprocess_progress_watchdog_interval ............ 1.0
|
| 239 |
+
inprocess_restart ............................... False
|
| 240 |
+
inprocess_soft_timeout .......................... 60
|
| 241 |
+
inprocess_termination_grace_time ................ 1
|
| 242 |
+
is_hybrid_model ................................. False
|
| 243 |
+
iter_per_epoch .................................. 1250
|
| 244 |
+
iterations_to_skip .............................. []
|
| 245 |
+
keep_fp8_transpose_cache_when_using_custom_fsdp . False
|
| 246 |
+
kitchen_config_file ............................. None
|
| 247 |
+
kitchen_recipe_number ........................... None
|
| 248 |
+
kv_channels ..................................... 64
|
| 249 |
+
kv_lora_rank .................................... 512
|
| 250 |
+
lazy_mpu_init ................................... None
|
| 251 |
+
load ............................................ None
|
| 252 |
+
load_main_params_from_ckpt ...................... None
|
| 253 |
+
load_model_opt_format ........................... False
|
| 254 |
+
local_rank ...................................... 0
|
| 255 |
+
log_energy ...................................... False
|
| 256 |
+
log_interval .................................... 1
|
| 257 |
+
log_loss_scale_to_tensorboard ................... True
|
| 258 |
+
log_memory_to_tensorboard ....................... False
|
| 259 |
+
log_num_zeros_in_grad ........................... False
|
| 260 |
+
log_params_norm ................................. False
|
| 261 |
+
log_progress .................................... False
|
| 262 |
+
log_straggler ................................... False
|
| 263 |
+
log_throughput .................................. True
|
| 264 |
+
log_timers_to_tensorboard ....................... False
|
| 265 |
+
log_validation_ppl_to_tensorboard ............... False
|
| 266 |
+
log_world_size_to_tensorboard ................... False
|
| 267 |
+
logging_level ................................... None
|
| 268 |
+
loss_scale ...................................... None
|
| 269 |
+
loss_scale_window ............................... 1000
|
| 270 |
+
lr .............................................. 0.0001
|
| 271 |
+
lr_decay_iters .................................. 8
|
| 272 |
+
lr_decay_samples ................................ None
|
| 273 |
+
lr_decay_style .................................. cosine
|
| 274 |
+
lr_warmup_fraction .............................. None
|
| 275 |
+
lr_warmup_init .................................. 0.0
|
| 276 |
+
lr_warmup_iters ................................. 5
|
| 277 |
+
lr_warmup_samples ............................... 0
|
| 278 |
+
lr_wsd_decay_iters .............................. None
|
| 279 |
+
lr_wsd_decay_samples ............................ None
|
| 280 |
+
lr_wsd_decay_style .............................. exponential
|
| 281 |
+
main_grads_dtype ................................ torch.float32
|
| 282 |
+
main_params_dtype ............................... torch.float32
|
| 283 |
+
make_vocab_size_divisible_by .................... 128
|
| 284 |
+
mamba_head_dim .................................. 64
|
| 285 |
+
mamba_num_groups ................................ 8
|
| 286 |
+
mamba_num_heads ................................. None
|
| 287 |
+
mamba_state_dim ................................. 128
|
| 288 |
+
manual_gc ....................................... False
|
| 289 |
+
manual_gc_eval .................................. True
|
| 290 |
+
manual_gc_interval .............................. 0
|
| 291 |
+
mask_factor ..................................... 1.0
|
| 292 |
+
mask_prob ....................................... 0.15
|
| 293 |
+
mask_type ....................................... random
|
| 294 |
+
masked_softmax_fusion ........................... False
|
| 295 |
+
max_position_embeddings ......................... 4096
|
| 296 |
+
max_tokens_to_oom ............................... 12000
|
| 297 |
+
memory_snapshot_path ............................ snapshot.pickle
|
| 298 |
+
merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt
|
| 299 |
+
micro_batch_size ................................ 1
|
| 300 |
+
microbatch_group_size_per_vp_stage .............. None
|
| 301 |
+
mid_level_dataset_surplus ....................... 0.005
|
| 302 |
+
min_loss_scale .................................. 1.0
|
| 303 |
+
min_lr .......................................... 1e-05
|
| 304 |
+
mlp_chunks_for_prefill .......................... 1
|
| 305 |
+
mmap_bin_files .................................. True
|
| 306 |
+
mock_data ....................................... False
|
| 307 |
+
moe_apply_probs_on_input ........................ False
|
| 308 |
+
moe_aux_loss_coeff .............................. 0.0
|
| 309 |
+
moe_capacity_monitor ............................ True
|
| 310 |
+
moe_deepep_num_sms .............................. 20
|
| 311 |
+
moe_echo_enable_random_offloading ............... False
|
| 312 |
+
moe_echo_expert_dispatch_overlap ................ False
|
| 313 |
+
moe_echo_expert_dispatcher_type ................. hybridep
|
| 314 |
+
moe_echo_recompute_expert_dispatch .............. False
|
| 315 |
+
moe_enable_deepep ............................... False
|
| 316 |
+
moe_enable_echo ................................. False
|
| 317 |
+
moe_enable_hybridep ............................. True
|
| 318 |
+
moe_enable_ultraep .............................. True
|
| 319 |
+
moe_expert_capacity_factor ...................... None
|
| 320 |
+
moe_extended_tp ................................. False
|
| 321 |
+
moe_ffn_hidden_size ............................. 2048
|
| 322 |
+
moe_grouped_gemm ................................ True
|
| 323 |
+
moe_hybridep_num_sms ............................ 16
|
| 324 |
+
moe_input_jitter_eps ............................ None
|
| 325 |
+
moe_layer_freq .................................. [0, 1]
|
| 326 |
+
moe_layer_recompute ............................. False
|
| 327 |
+
moe_num_echo_experts ............................ None
|
| 328 |
+
moe_num_redundant_experts_per_rank .............. 48
|
| 329 |
+
moe_pad_expert_input_to_capacity ................ False
|
| 330 |
+
moe_per_layer_logging ........................... False
|
| 331 |
+
moe_permute_fusion .............................. False
|
| 332 |
+
moe_received_token_capacity ..................... 1.17578125
|
| 333 |
+
moe_router_bias_update_rate ..................... 0.001
|
| 334 |
+
moe_router_dtype ................................ fp32
|
| 335 |
+
moe_router_enable_expert_bias ................... True
|
| 336 |
+
moe_router_force_load_balancing ................. False
|
| 337 |
+
moe_router_group_topk ........................... 1
|
| 338 |
+
moe_router_load_balancing_type .................. seq_aux_loss
|
| 339 |
+
moe_router_num_groups ........................... 1
|
| 340 |
+
moe_router_padding_for_fp8 ...................... False
|
| 341 |
+
moe_router_pre_softmax .......................... False
|
| 342 |
+
moe_router_score_function ....................... sigmoid
|
| 343 |
+
moe_router_topk ................................. 8
|
| 344 |
+
moe_router_topk_scaling_factor .................. 2.827
|
| 345 |
+
moe_shared_expert_intermediate_size ............. 2048
|
| 346 |
+
moe_shared_expert_overlap ....................... False
|
| 347 |
+
moe_token_dispatcher_type ....................... flex
|
| 348 |
+
moe_token_drop_policy ........................... probs
|
| 349 |
+
moe_ultraep_max_microbatches .................... None
|
| 350 |
+
moe_ultraep_overlap_weight_sync ................. True
|
| 351 |
+
moe_ultraep_weight_sync_join .................... before_dispatch
|
| 352 |
+
moe_upcycling_granularity ....................... 1
|
| 353 |
+
moe_use_device_initiated_grouped_gemm ........... True
|
| 354 |
+
moe_use_legacy_grouped_gemm ..................... False
|
| 355 |
+
moe_use_upcycling ............................... False
|
| 356 |
+
moe_z_loss_coeff ................................ None
|
| 357 |
+
mrope_section ................................... None
|
| 358 |
+
mscale .......................................... 1.0
|
| 359 |
+
mscale_all_dim .................................. 1.0
|
| 360 |
+
mtp_loss_scaling_factor ......................... 0.1
|
| 361 |
+
mtp_num_layers .................................. None
|
| 362 |
+
multi_latent_attention .......................... True
|
| 363 |
+
nccl_all_reduce_for_prefill ..................... False
|
| 364 |
+
nccl_communicator_config_path ................... None
|
| 365 |
+
nccl_ub ......................................... False
|
| 366 |
+
no_load_optim ................................... None
|
| 367 |
+
no_load_rng ..................................... None
|
| 368 |
+
no_persist_layer_norm ........................... False
|
| 369 |
+
no_rope_freq .................................... None
|
| 370 |
+
no_save_optim ................................... None
|
| 371 |
+
no_save_rng ..................................... None
|
| 372 |
+
non_persistent_ckpt_type ........................ None
|
| 373 |
+
non_persistent_global_ckpt_dir .................. None
|
| 374 |
+
non_persistent_local_ckpt_algo .................. fully_parallel
|
| 375 |
+
non_persistent_local_ckpt_dir ................... None
|
| 376 |
+
non_persistent_save_interval .................... None
|
| 377 |
+
norm_epsilon .................................... 1e-06
|
| 378 |
+
normalization ................................... RMSNorm
|
| 379 |
+
num_attention_heads ............................. 64
|
| 380 |
+
num_channels .................................... 3
|
| 381 |
+
num_classes ..................................... 1000
|
| 382 |
+
num_dataset_builder_threads ..................... 1
|
| 383 |
+
num_distributed_optimizer_instances ............. 1
|
| 384 |
+
num_experts ..................................... 384
|
| 385 |
+
num_layers ...................................... 2
|
| 386 |
+
num_layers_at_end_in_bf16 ....................... 1
|
| 387 |
+
num_layers_at_start_in_bf16 ..................... 1
|
| 388 |
+
num_layers_per_virtual_pipeline_stage ........... None
|
| 389 |
+
num_query_groups ................................ 1
|
| 390 |
+
num_virtual_stages_per_pipeline_rank ............ None
|
| 391 |
+
num_workers ..................................... 2
|
| 392 |
+
object_storage_cache_path ....................... None
|
| 393 |
+
one_logger_async ................................ False
|
| 394 |
+
one_logger_project .............................. megatron-lm
|
| 395 |
+
one_logger_run_name ............................. None
|
| 396 |
+
onnx_safe ....................................... None
|
| 397 |
+
openai_gelu ..................................... False
|
| 398 |
+
optimizer ....................................... adam
|
| 399 |
+
optimizer_cpu_offload ........................... False
|
| 400 |
+
optimizer_offload_fraction ...................... 1.0
|
| 401 |
+
output_bert_embeddings .......................... False
|
| 402 |
+
overlap_cpu_optimizer_d2h_h2d ................... False
|
| 403 |
+
overlap_grad_reduce ............................. True
|
| 404 |
+
overlap_moe_expert_parallel_comm ................ False
|
| 405 |
+
overlap_p2p_comm ................................ False
|
| 406 |
+
overlap_p2p_comm_warmup_flush ................... False
|
| 407 |
+
overlap_param_gather ............................ True
|
| 408 |
+
overlap_param_gather_with_optimizer_step ........ False
|
| 409 |
+
override_opt_param_scheduler .................... False
|
| 410 |
+
params_dtype .................................... torch.bfloat16
|
| 411 |
+
patch_dim ....................................... 16
|
| 412 |
+
per_split_data_args_path ........................ None
|
| 413 |
+
perform_initialization .......................... True
|
| 414 |
+
pin_cpu_grads ................................... True
|
| 415 |
+
pin_cpu_params .................................. True
|
| 416 |
+
pipeline_model_parallel_comm_backend ............ None
|
| 417 |
+
pipeline_model_parallel_layout .................. None
|
| 418 |
+
pipeline_model_parallel_size .................... 1
|
| 419 |
+
position_embedding_type ......................... rope
|
| 420 |
+
pretrained_checkpoint ........................... None
|
| 421 |
+
profile ......................................... False
|
| 422 |
+
profile_ranks ................................... [0]
|
| 423 |
+
profile_step_end ................................ 12
|
| 424 |
+
profile_step_start .............................. 10
|
| 425 |
+
q_lora_rank ..................................... 1536
|
| 426 |
+
qk_head_dim ..................................... 128
|
| 427 |
+
qk_l2_norm ...................................... False
|
| 428 |
+
qk_layernorm .................................... True
|
| 429 |
+
qk_pos_emb_head_dim ............................. 64
|
| 430 |
+
query_in_block_prob ............................. 0.1
|
| 431 |
+
rampup_batch_size ............................... None
|
| 432 |
+
rank ............................................ 0
|
| 433 |
+
recompute_granularity ........................... None
|
| 434 |
+
recompute_method ................................ None
|
| 435 |
+
recompute_modules ............................... None
|
| 436 |
+
recompute_num_layers ............................ None
|
| 437 |
+
record_memory_history ........................... False
|
| 438 |
+
relative_attention_max_distance ................. 128
|
| 439 |
+
relative_attention_num_buckets .................. 32
|
| 440 |
+
replication ..................................... False
|
| 441 |
+
replication_factor .............................. 2
|
| 442 |
+
replication_jump ................................ None
|
| 443 |
+
rerun_mode ...................................... validate_results
|
| 444 |
+
reset_attention_mask ............................ False
|
| 445 |
+
reset_position_ids .............................. False
|
| 446 |
+
result_rejected_tracker_filename ................ None
|
| 447 |
+
retriever_report_topk_accuracies ................ []
|
| 448 |
+
retriever_score_scaling ......................... False
|
| 449 |
+
retriever_seq_length ............................ 256
|
| 450 |
+
retro_add_retriever ............................. False
|
| 451 |
+
retro_attention_gate ............................ 1
|
| 452 |
+
retro_cyclic_train_iters ........................ None
|
| 453 |
+
retro_encoder_attention_dropout ................. 0.1
|
| 454 |
+
retro_encoder_hidden_dropout .................... 0.1
|
| 455 |
+
retro_encoder_layers ............................ 2
|
| 456 |
+
retro_num_neighbors ............................. 2
|
| 457 |
+
retro_num_retrieved_chunks ...................... 2
|
| 458 |
+
retro_project_dir ............................... None
|
| 459 |
+
retro_verify_neighbor_count ..................... True
|
| 460 |
+
reuse_grad_buf_for_mxfp8_param_ag ............... False
|
| 461 |
+
rope_scaling_factor ............................. 8.0
|
| 462 |
+
rope_type ....................................... None
|
| 463 |
+
rotary_base ..................................... 50000
|
| 464 |
+
rotary_interleaved .............................. False
|
| 465 |
+
rotary_percent .................................. 1.0
|
| 466 |
+
rotary_scaling_factor ........................... 32.0
|
| 467 |
+
rotary_seq_len_interpolation_factor ............. None
|
| 468 |
+
run_workload_inspector_server ................... False
|
| 469 |
+
sample_rate ..................................... 1.0
|
| 470 |
+
save ............................................ None
|
| 471 |
+
save_interval ................................... None
|
| 472 |
+
save_retain_interval ............................ None
|
| 473 |
+
scatter_gather_tensors_in_pipeline .............. True
|
| 474 |
+
seed ............................................ 1234
|
| 475 |
+
seq_length ...................................... 4096
|
| 476 |
+
sequence_parallel ............................... False
|
| 477 |
+
sft ............................................. False
|
| 478 |
+
sft_tokenizer_prompt_format ..................... nemotron-h-aligned
|
| 479 |
+
sgd_momentum .................................... 0.9
|
| 480 |
+
short_seq_prob .................................. 0.1
|
| 481 |
+
skip_train ...................................... False
|
| 482 |
+
skipped_train_samples ........................... 0
|
| 483 |
+
spec ............................................ None
|
| 484 |
+
split ........................................... 100,0,0
|
| 485 |
+
squared_relu .................................... False
|
| 486 |
+
start_weight_decay .............................. 0.1
|
| 487 |
+
straggler_ctrlr_port ............................ 65535
|
| 488 |
+
straggler_minmax_count .......................... 1
|
| 489 |
+
suggested_communication_unit_size ............... None
|
| 490 |
+
swiglu .......................................... True
|
| 491 |
+
swin_backbone_type .............................. tiny
|
| 492 |
+
symmetric_ar_type ............................... None
|
| 493 |
+
te_rng_tracker .................................. True
|
| 494 |
+
tensor_model_parallel_size ...................... 1
|
| 495 |
+
tensorboard_dir ................................. None
|
| 496 |
+
tensorboard_log_interval ........................ 1
|
| 497 |
+
tensorboard_queue_size .......................... 1000
|
| 498 |
+
test_data_path .................................. None
|
| 499 |
+
test_mode ....................................... False
|
| 500 |
+
tiktoken_num_special_tokens ..................... 1000
|
| 501 |
+
tiktoken_pattern ................................ None
|
| 502 |
+
tiktoken_special_tokens ......................... None
|
| 503 |
+
timing_log_level ................................ 0
|
| 504 |
+
timing_log_option ............................... minmax
|
| 505 |
+
titles_data_path ................................ None
|
| 506 |
+
tokenizer_model ................................. None
|
| 507 |
+
tokenizer_type .................................. GPT2BPETokenizer
|
| 508 |
+
torch_fsdp2_reshard_after_forward ............... True
|
| 509 |
+
tp_comm_bootstrap_backend ....................... nccl
|
| 510 |
+
tp_comm_bulk_dgrad .............................. True
|
| 511 |
+
tp_comm_bulk_wgrad .............................. True
|
| 512 |
+
tp_comm_overlap ................................. False
|
| 513 |
+
tp_comm_overlap_ag .............................. True
|
| 514 |
+
tp_comm_overlap_cfg ............................. None
|
| 515 |
+
tp_comm_overlap_rs .............................. True
|
| 516 |
+
tp_comm_overlap_rs_dgrad ........................ False
|
| 517 |
+
tp_comm_split_ag ................................ True
|
| 518 |
+
tp_comm_split_rs ................................ True
|
| 519 |
+
train_data_path ................................. None
|
| 520 |
+
train_iters ..................................... 12
|
| 521 |
+
train_samples ................................... None
|
| 522 |
+
train_sync_interval ............................. None
|
| 523 |
+
transformer_impl ................................ transformer_engine
|
| 524 |
+
transformer_pipeline_model_parallel_size ........ 1
|
| 525 |
+
untie_embeddings_and_output_weights ............. True
|
| 526 |
+
use_checkpoint_args ............................. False
|
| 527 |
+
use_checkpoint_opt_param_scheduler .............. False
|
| 528 |
+
use_cpu_initialization .......................... None
|
| 529 |
+
use_custom_fsdp ................................. False
|
| 530 |
+
use_dist_ckpt ................................... True
|
| 531 |
+
use_dist_ckpt_deprecated ........................ False
|
| 532 |
+
use_distributed_optimizer ....................... True
|
| 533 |
+
use_flash_attn .................................. False
|
| 534 |
+
use_legacy_models ............................... False
|
| 535 |
+
use_mp_args_from_checkpoint_args ................ False
|
| 536 |
+
use_one_sent_docs ............................... False
|
| 537 |
+
use_persistent_ckpt_worker ...................... False
|
| 538 |
+
use_precision_aware_optimizer ................... False
|
| 539 |
+
use_pytorch_profiler ............................ False
|
| 540 |
+
use_ring_exchange_p2p ........................... False
|
| 541 |
+
use_rope_scaling ................................ False
|
| 542 |
+
use_rotary_position_embeddings .................. False
|
| 543 |
+
use_sharp ....................................... False
|
| 544 |
+
use_tokenizer_model_from_checkpoint_args ........ True
|
| 545 |
+
use_torch_fsdp2 ................................. False
|
| 546 |
+
use_torch_optimizer_for_cpu_offload ............. False
|
| 547 |
+
use_tp_pp_dp_mapping ............................ False
|
| 548 |
+
v_head_dim ...................................... 128
|
| 549 |
+
valid_data_path ................................. None
|
| 550 |
+
variable_seq_lengths ............................ False
|
| 551 |
+
virtual_pipeline_model_parallel_size ............ None
|
| 552 |
+
vision_backbone_type ............................ vit
|
| 553 |
+
vision_pretraining .............................. False
|
| 554 |
+
vision_pretraining_type ......................... classify
|
| 555 |
+
vocab_extra_ids ................................. 0
|
| 556 |
+
vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json
|
| 557 |
+
vocab_size ...................................... None
|
| 558 |
+
wandb_exp_name ..................................
|
| 559 |
+
wandb_project ...................................
|
| 560 |
+
wandb_save_dir ..................................
|
| 561 |
+
weight_decay .................................... 0.1
|
| 562 |
+
weight_decay_incr_style ......................... constant
|
| 563 |
+
wgrad_deferral_limit ............................ 0
|
| 564 |
+
world_size ...................................... 8
|
| 565 |
+
yaml_cfg ........................................ None
|
| 566 |
+
-------------------- end of arguments ---------------------
|
| 567 |
+
INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1
|
| 568 |
+
> building GPT2BPETokenizer tokenizer ...
|
| 569 |
+
> padded vocab (size: 50257) with 47 dummy tokens (new size: 50304)
|
| 570 |
+
WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS
|
| 571 |
+
> initializing torch distributed ...
|
| 572 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 573 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 574 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 575 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 576 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 577 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 578 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 579 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 580 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 581 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 582 |
+
WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it
|
| 583 |
+
[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 584 |
+
[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank
|
| 585 |
+
3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 586 |
+
[Gloo] Rank [Gloo] Rank 7 is connected to 75[Gloo] Rank peer ranks. is connected to 2Expected number of connected peer ranks is : 7 is connected to 7 peer ranks. 7
|
| 587 |
+
Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is :
|
| 588 |
+
7
|
| 589 |
+
[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 590 |
+
[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 591 |
+
[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 592 |
+
[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 593 |
+
[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 594 |
+
[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 595 |
+
[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 596 |
+
[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 597 |
+
[Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is :
|
| 598 |
+
7
|
| 599 |
+
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
|
| 600 |
+
> initialized tensor model parallel with size 1
|
| 601 |
+
> initialized pipeline model parallel with size 1
|
| 602 |
+
> setting random seeds to 1234 ...
|
| 603 |
+
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
|
| 604 |
+
[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00
|
| 605 |
+
|
| 606 |
+
[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00
|
| 607 |
+
|
| 608 |
+
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
|
| 609 |
+
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
|
| 610 |
+
[rank5]:[W921 07:47:45.193572765 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 611 |
+
[rank2]:[W921 07:47:45.193626668 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 612 |
+
[rank7]:[W921 07:47:45.195412548 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 613 |
+
[rank4]:[W921 07:47:45.195476768 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 614 |
+
[rank3]:[W921 07:47:45.195480188 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 615 |
+
[rank1]:[W921 07:47:45.195505767 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 616 |
+
[rank6]:[W921 07:47:45.195651183 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 617 |
+
> compiling dataset index builder ...
|
| 618 |
+
make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
|
| 619 |
+
make: Nothing to be done for 'default'.
|
| 620 |
+
make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
|
| 621 |
+
>>> done with dataset index builder. Compilation time: 0.035 seconds
|
| 622 |
+
WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations.
|
| 623 |
+
> compiling and loading fused kernels ...
|
| 624 |
+
[rank0]:[W921 07:47:45.354250232 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 625 |
+
>>> done with compiling and loading fused kernels. Compilation time: 3.644 seconds
|
| 626 |
+
time to initialize megatron (seconds): 6.952
|
| 627 |
+
[after megatron is initialized] datetime: 2026-09-21 07:47:51
|
| 628 |
+
building GPT model ...
|
| 629 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 630 |
+
warnings.warn(
|
| 631 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 632 |
+
warnings.warn(
|
| 633 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 634 |
+
warnings.warn(
|
| 635 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 636 |
+
warnings.warn(
|
| 637 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 638 |
+
warnings.warn(
|
| 639 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 640 |
+
warnings.warn(
|
| 641 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 642 |
+
warnings.warn(
|
| 643 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 644 |
+
warnings.warn(
|
| 645 |
+
[INFO] Use MNNVL fabric: False
|
| 646 |
+
[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8.
|
| 647 |
+
NVSHMEM v3.7.2
|
| 648 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 649 |
+
|
| 650 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 651 |
+
|
| 652 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 653 |
+
|
| 654 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 655 |
+
|
| 656 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 657 |
+
|
| 658 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 659 |
+
|
| 660 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 661 |
+
|
| 662 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 663 |
+
|
| 664 |
+
UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive).
|
| 665 |
+
> number of parameters on (tensor, pipeline) model parallel rank (0, 0): 5594455040
|
| 666 |
+
INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False)
|
| 667 |
+
INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 9
|
| 668 |
+
Params for bucket 1 (360579072 elements, 360579072 padded size):
|
| 669 |
+
module.output_layer.weight
|
| 670 |
+
Params for bucket 2 (44047360 elements, 44047360 padded size):
|
| 671 |
+
module.decoder.final_layernorm.weight
|
| 672 |
+
module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight
|
| 673 |
+
module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight
|
| 674 |
+
Params for bucket 3 (45163520 elements, 45163520 padded size):
|
| 675 |
+
module.decoder.layers.1.mlp.router.weight
|
| 676 |
+
module.decoder.layers.1.self_attention.linear_q_up_proj.layer_norm_weight
|
| 677 |
+
module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight
|
| 678 |
+
module.decoder.layers.1.self_attention.linear_kv_down_proj.weight
|
| 679 |
+
module.decoder.layers.1.self_attention.linear_kv_up_proj.weight
|
| 680 |
+
module.decoder.layers.1.pre_mlp_layernorm.weight
|
| 681 |
+
module.decoder.layers.1.self_attention.linear_q_up_proj.weight
|
| 682 |
+
module.decoder.layers.1.self_attention.linear_q_down_proj.weight
|
| 683 |
+
Params for bucket 4 (58720256 elements, 58720256 padded size):
|
| 684 |
+
module.decoder.layers.1.self_attention.linear_proj.weight
|
| 685 |
+
Params for bucket 5 (132127744 elements, 132127744 padded size):
|
| 686 |
+
module.decoder.layers.1.input_layernorm.weight
|
| 687 |
+
module.decoder.layers.0.mlp.linear_fc2.weight
|
| 688 |
+
Params for bucket 6 (264241152 elements, 264241152 padded size):
|
| 689 |
+
module.decoder.layers.0.mlp.linear_fc1.weight
|
| 690 |
+
Params for bucket 7 (42411008 elements, 42411008 padded size):
|
| 691 |
+
module.decoder.layers.0.self_attention.linear_kv_up_proj.weight
|
| 692 |
+
module.decoder.layers.0.self_attention.linear_q_up_proj.layer_norm_weight
|
| 693 |
+
module.decoder.layers.0.self_attention.linear_q_down_proj.weight
|
| 694 |
+
module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight
|
| 695 |
+
module.decoder.layers.0.self_attention.linear_kv_down_proj.weight
|
| 696 |
+
module.decoder.layers.0.self_attention.linear_q_up_proj.weight
|
| 697 |
+
module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight
|
| 698 |
+
Params for bucket 8 (58720256 elements, 58720256 padded size):
|
| 699 |
+
module.decoder.layers.0.self_attention.linear_proj.weight
|
| 700 |
+
Params for bucket 9 (360586240 elements, 360586240 padded size):
|
| 701 |
+
module.decoder.layers.0.input_layernorm.weight
|
| 702 |
+
module.embedding.word_embeddings.weight
|
| 703 |
+
INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40
|
| 704 |
+
Params for bucket 1 (44040192 elements, 44040192 padded size):
|
| 705 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight47
|
| 706 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight45
|
| 707 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight46
|
| 708 |
+
Params for bucket 2 (44040192 elements, 44040192 padded size):
|
| 709 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight44
|
| 710 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight42
|
| 711 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight43
|
| 712 |
+
Params for bucket 3 (44040192 elements, 44040192 padded size):
|
| 713 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight41
|
| 714 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight39
|
| 715 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight40
|
| 716 |
+
Params for bucket 4 (44040192 elements, 44040192 padded size):
|
| 717 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight38
|
| 718 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight36
|
| 719 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight37
|
| 720 |
+
Params for bucket 5 (44040192 elements, 44040192 padded size):
|
| 721 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight35
|
| 722 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight33
|
| 723 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight34
|
| 724 |
+
Params for bucket 6 (44040192 elements, 44040192 padded size):
|
| 725 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight32
|
| 726 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight31
|
| 727 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight30
|
| 728 |
+
Params for bucket 7 (44040192 elements, 44040192 padded size):
|
| 729 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight29
|
| 730 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight28
|
| 731 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight27
|
| 732 |
+
Params for bucket 8 (44040192 elements, 44040192 padded size):
|
| 733 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight26
|
| 734 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight25
|
| 735 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight24
|
| 736 |
+
Params for bucket 9 (44040192 elements, 44040192 padded size):
|
| 737 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight23
|
| 738 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight22
|
| 739 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight21
|
| 740 |
+
Params for bucket 10 (44040192 elements, 44040192 padded size):
|
| 741 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight20
|
| 742 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight19
|
| 743 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight18
|
| 744 |
+
Params for bucket 11 (44040192 elements, 44040192 padded size):
|
| 745 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight17
|
| 746 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight16
|
| 747 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight15
|
| 748 |
+
Params for bucket 12 (44040192 elements, 44040192 padded size):
|
| 749 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight14
|
| 750 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight13
|
| 751 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight12
|
| 752 |
+
Params for bucket 13 (44040192 elements, 44040192 padded size):
|
| 753 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight11
|
| 754 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight9
|
| 755 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight10
|
| 756 |
+
Params for bucket 14 (44040192 elements, 44040192 padded size):
|
| 757 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight8
|
| 758 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight6
|
| 759 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight7
|
| 760 |
+
Params for bucket 15 (44040192 elements, 44040192 padded size):
|
| 761 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight5
|
| 762 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight3
|
| 763 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight4
|
| 764 |
+
Params for bucket 16 (44040192 elements, 44040192 padded size):
|
| 765 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight2
|
| 766 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight0
|
| 767 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight1
|
| 768 |
+
Params for bucket 17 (58720256 elements, 58720256 padded size):
|
| 769 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight47
|
| 770 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight46
|
| 771 |
+
Params for bucket 18 (58720256 elements, 58720256 padded size):
|
| 772 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight45
|
| 773 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight44
|
| 774 |
+
Params for bucket 19 (58720256 elements, 58720256 padded size):
|
| 775 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight43
|
| 776 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight42
|
| 777 |
+
Params for bucket 20 (58720256 elements, 58720256 padded size):
|
| 778 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight41
|
| 779 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight40
|
| 780 |
+
Params for bucket 21 (58720256 elements, 58720256 padded size):
|
| 781 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight39
|
| 782 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight38
|
| 783 |
+
Params for bucket 22 (58720256 elements, 58720256 padded size):
|
| 784 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight37
|
| 785 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight36
|
| 786 |
+
Params for bucket 23 (58720256 elements, 58720256 padded size):
|
| 787 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight35
|
| 788 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight34
|
| 789 |
+
Params for bucket 24 (58720256 elements, 58720256 padded size):
|
| 790 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight33
|
| 791 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight32
|
| 792 |
+
Params for bucket 25 (58720256 elements, 58720256 padded size):
|
| 793 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight31
|
| 794 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight30
|
| 795 |
+
Params for bucket 26 (58720256 elements, 58720256 padded size):
|
| 796 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight29
|
| 797 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight28
|
| 798 |
+
Params for bucket 27 (58720256 elements, 58720256 padded size):
|
| 799 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight27
|
| 800 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight26
|
| 801 |
+
Params for bucket 28 (58720256 elements, 58720256 padded size):
|
| 802 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight25
|
| 803 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight24
|
| 804 |
+
Params for bucket 29 (58720256 elements, 58720256 padded size):
|
| 805 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight23
|
| 806 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight22
|
| 807 |
+
Params for bucket 30 (58720256 elements, 58720256 padded size):
|
| 808 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight21
|
| 809 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight20
|
| 810 |
+
Params for bucket 31 (58720256 elements, 58720256 padded size):
|
| 811 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight19
|
| 812 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight18
|
| 813 |
+
Params for bucket 32 (58720256 elements, 58720256 padded size):
|
| 814 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight17
|
| 815 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight16
|
| 816 |
+
Params for bucket 33 (58720256 elements, 58720256 padded size):
|
| 817 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight15
|
| 818 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight14
|
| 819 |
+
Params for bucket 34 (58720256 elements, 58720256 padded size):
|
| 820 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight13
|
| 821 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight12
|
| 822 |
+
Params for bucket 35 (58720256 elements, 58720256 padded size):
|
| 823 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight11
|
| 824 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight10
|
| 825 |
+
Params for bucket 36 (58720256 elements, 58720256 padded size):
|
| 826 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight9
|
| 827 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight8
|
| 828 |
+
Params for bucket 37 (58720256 elements, 58720256 padded size):
|
| 829 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight7
|
| 830 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight6
|
| 831 |
+
Params for bucket 38 (58720256 elements, 58720256 padded size):
|
| 832 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight5
|
| 833 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight4
|
| 834 |
+
Params for bucket 39 (58720256 elements, 58720256 padded size):
|
| 835 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight3
|
| 836 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight2
|
| 837 |
+
Params for bucket 40 (58720256 elements, 58720256 padded size):
|
| 838 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight1
|
| 839 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight0
|
| 840 |
+
INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=<megatron.core.timers.Timers object at 0x72c41e8fea20>, config_logger_dir='')
|
| 841 |
+
INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine
|
| 842 |
+
[after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-21 07:47:54
|
| 843 |
+
> building train, validation, and test datasets ...
|
| 844 |
+
> datasets target sizes (minimum size):
|
| 845 |
+
train: 96
|
| 846 |
+
validation: 0
|
| 847 |
+
test: 0
|
| 848 |
+
INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None]
|
| 849 |
+
> building train, validation, and test datasets for GPT ...
|
| 850 |
+
INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=<megatron.training.tokenizer.tokenizer._GPT2BPETokenizer object at 0x72c41edc2000>, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None)
|
| 851 |
+
INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx
|
| 852 |
+
INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths
|
| 853 |
+
INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers
|
| 854 |
+
INFO:megatron.core.datasets.indexed_dataset: Extract the document indices
|
| 855 |
+
INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029
|
| 856 |
+
INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350
|
| 857 |
+
INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices
|
| 858 |
+
INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy
|
| 859 |
+
INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy
|
| 860 |
+
INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy
|
| 861 |
+
INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073
|
| 862 |
+
> finished creating GPT datasets ...
|
| 863 |
+
[after dataloaders are built] datetime: 2026-09-21 07:47:54
|
| 864 |
+
done with setup ...
|
| 865 |
+
(min, max) time across ranks (ms):
|
| 866 |
+
model-and-optimizer-setup ......................: (3380.12, 3384.53)
|
| 867 |
+
train/valid/test-data-iterators-setup ..........: (72.90, 94.49)
|
| 868 |
+
training ...
|
| 869 |
+
Overwriting rerun_state_machine.current_iteration from -1 to 0...
|
| 870 |
+
[before the start of training step] datetime: 2026-09-21 07:47:54
|
| 871 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 872 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 873 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 874 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 875 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 876 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 877 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 878 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 879 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 880 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 881 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 882 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 883 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 884 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 885 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 886 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 887 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 888 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 889 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 890 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 891 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 892 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 893 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 894 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 895 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 896 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 897 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 898 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 899 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 900 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 901 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 902 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 903 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 904 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 905 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 906 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 907 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 908 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 909 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 910 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 911 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 912 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 913 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 914 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 915 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 916 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 917 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 918 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 919 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 920 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 921 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 922 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 923 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 924 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 925 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 926 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 927 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 928 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 929 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 930 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 931 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 932 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 933 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 934 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 935 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 936 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 937 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 938 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 939 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 940 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 941 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 942 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 943 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 944 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 945 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 946 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 947 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 948 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 949 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 950 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 951 |
+
[2026-09-21 07:48:22] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 28037.9 | throughput per GPU (TFLOP/s/GPU): 1.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.227924E+01 | loss scale: 1.0 | grad norm: 12.617 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 |
|
| 952 |
+
Number of parameters in transformer block in billions: 17.55
|
| 953 |
+
Number of parameters in embedding layers in billions: 0.72
|
| 954 |
+
Total number of parameters in billions: 18.28
|
| 955 |
+
Number of parameters in most loaded shard in billions: 18.2753
|
| 956 |
+
Theoretical memory footprints: weight and optimizer=130715.23 MB
|
| 957 |
+
[Rank 0] (after 1 iterations) memory (MB) | allocated: 49136.10400390625 | max allocated: 49136.1044921875 | reserved: 54394.0 | max reserved: 54394.0
|
| 958 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 959 |
+
warnings.warn(
|
| 960 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 961 |
+
warnings.warn(
|
| 962 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 963 |
+
warnings.warn(
|
| 964 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 965 |
+
warnings.warn(
|
| 966 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 967 |
+
warnings.warn(
|
| 968 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 969 |
+
warnings.warn(
|
| 970 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 971 |
+
warnings.warn(
|
| 972 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 973 |
+
warnings.warn(
|
| 974 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 975 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 976 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 977 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 978 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 979 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 980 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 981 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 982 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 983 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 984 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 985 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 986 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 987 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 988 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 989 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 990 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 991 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 992 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 993 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 994 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 995 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 996 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 997 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 998 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 999 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1000 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 1001 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 1002 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1003 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1004 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 1005 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 1006 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1007 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1008 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1009 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1010 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1011 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1012 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1013 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1014 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1015 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1016 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1017 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1018 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1019 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1020 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1021 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1022 |
+
[2026-09-21 07:48:22] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 163.2 | throughput per GPU (TFLOP/s/GPU): 216.7 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.227088E+01 | loss scale: 1.0 | grad norm: 12.534 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 |
|
| 1023 |
+
[2026-09-21 07:48:22] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 111.8 | throughput per GPU (TFLOP/s/GPU): 316.4 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.088328E+01 | loss scale: 1.0 | grad norm: 11.023 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 |
|
| 1024 |
+
Capture CUDA graph for training!!!
|
| 1025 |
+
CUDA graph capture done!!!
|
| 1026 |
+
[2026-09-21 07:48:24] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 1313.7 | throughput per GPU (TFLOP/s/GPU): 26.9 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.448181E+01 | loss scale: 1.0 | grad norm: 72.911 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36608 | min safe capacity: 1.1172 |
|
| 1027 |
+
[2026-09-21 07:48:24] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 110.4 | throughput per GPU (TFLOP/s/GPU): 320.3 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.808459E+01 | loss scale: 1.0 | grad norm: 108.268 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1028 |
+
[2026-09-21 07:48:24] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 111.9 | throughput per GPU (TFLOP/s/GPU): 316.2 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.491727E+01 | loss scale: 1.0 | grad norm: 83.473 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1029 |
+
[2026-09-21 07:48:24] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 114.9 | throughput per GPU (TFLOP/s/GPU): 307.9 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.143662E+01 | loss scale: 1.0 | grad norm: 26.627 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1030 |
+
[2026-09-21 07:48:24] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 108.6 | throughput per GPU (TFLOP/s/GPU): 325.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.166943E+01 | loss scale: 1.0 | grad norm: 12.800 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1031 |
+
[2026-09-21 07:48:24] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 109.1 | throughput per GPU (TFLOP/s/GPU): 324.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.131635E+01 | loss scale: 1.0 | grad norm: 10.441 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1032 |
+
[2026-09-21 07:48:24] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 108.9 | throughput per GPU (TFLOP/s/GPU): 324.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.096943E+01 | loss scale: 1.0 | grad norm: 9.147 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1033 |
+
[2026-09-21 07:48:24] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 108.2 | throughput per GPU (TFLOP/s/GPU): 327.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.073911E+01 | loss scale: 1.0 | grad norm: 9.755 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1034 |
+
[2026-09-21 07:48:25] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 108.8 | throughput per GPU (TFLOP/s/GPU): 325.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.048286E+01 | loss scale: 1.0 | grad norm: 8.843 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1035 |
+
[after training is done] datetime: 2026-09-21 07:48:25
|
| 1036 |
+
[rank0]:[W921 07:48:26.297949340 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
|
| 1037 |
+
W0921 07:48:26.937000 519691 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers
|
| 1038 |
+
W0921 07:48:26.938000 519691 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 519990 closing signal SIGTERM
|
| 1039 |
+
W0921 07:48:26.938000 519691 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 519991 closing signal SIGTERM
|
| 1040 |
+
W0921 07:48:26.939000 519691 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 519993 closing signal SIGTERM
|
| 1041 |
+
W0921 07:48:26.940000 519691 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 519994 closing signal SIGTERM
|
| 1042 |
+
W0921 07:48:26.941000 519691 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 519996 closing signal SIGTERM
|
| 1043 |
+
W0921 07:48:26.942000 519691 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 519997 closing signal SIGTERM
|
| 1044 |
+
W0921 07:48:26.943000 519691 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 519998 closing signal SIGTERM
|
| 1045 |
+
W0921 07:48:26.944000 519691 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 519999 closing signal SIGTERM
|
| 1046 |
+
Traceback (most recent call last):
|
| 1047 |
+
File "/usr/local/bin/torchrun", line 8, in <module>
|
| 1048 |
+
sys.exit(main())
|
| 1049 |
+
^^^^^^
|
| 1050 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
|
| 1051 |
+
return f(*args, **kwargs)
|
| 1052 |
+
^^^^^^^^^^^^^^^^^^
|
| 1053 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main
|
| 1054 |
+
run(args)
|
| 1055 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run
|
| 1056 |
+
elastic_launch(
|
| 1057 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__
|
| 1058 |
+
return launch_agent(self._config, self._entrypoint, list(args))
|
| 1059 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 1060 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent
|
| 1061 |
+
result = agent.run()
|
| 1062 |
+
^^^^^^^^^^^
|
| 1063 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
| 1064 |
+
result = f(*args, **kwargs)
|
| 1065 |
+
^^^^^^^^^^^^^^^^^^
|
| 1066 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run
|
| 1067 |
+
result = self._invoke_run(role)
|
| 1068 |
+
^^^^^^^^^^^^^^^^^^^^^^
|
| 1069 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run
|
| 1070 |
+
time.sleep(monitor_interval)
|
| 1071 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler
|
| 1072 |
+
raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
|
| 1073 |
+
torch.distributed.elastic.multiprocessing.api.SignalException: Process 519691 got signal: 15
|
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep2/mem_samples.csv
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
0, 0
|
| 2 |
+
1, 0
|
| 3 |
+
2, 0
|
| 4 |
+
3, 0
|
| 5 |
+
4, 0
|
| 6 |
+
5, 0
|
| 7 |
+
6, 0
|
| 8 |
+
7, 0
|
| 9 |
+
0, 0
|
| 10 |
+
1, 0
|
| 11 |
+
2, 0
|
| 12 |
+
3, 0
|
| 13 |
+
4, 0
|
| 14 |
+
5, 0
|
| 15 |
+
6, 0
|
| 16 |
+
7, 0
|
| 17 |
+
0, 4
|
| 18 |
+
1, 1
|
| 19 |
+
2, 1
|
| 20 |
+
3, 1
|
| 21 |
+
4, 1
|
| 22 |
+
5, 1
|
| 23 |
+
6, 1
|
| 24 |
+
7, 1
|
| 25 |
+
0, 4
|
| 26 |
+
1, 4
|
| 27 |
+
2, 4
|
| 28 |
+
3, 4
|
| 29 |
+
4, 1
|
| 30 |
+
5, 1
|
| 31 |
+
6, 1
|
| 32 |
+
7, 1
|
| 33 |
+
0, 4
|
| 34 |
+
1, 4
|
| 35 |
+
2, 4
|
| 36 |
+
3, 4
|
| 37 |
+
4, 4
|
| 38 |
+
5, 4
|
| 39 |
+
6, 4
|
| 40 |
+
7, 4
|
| 41 |
+
0, 4
|
| 42 |
+
1, 629
|
| 43 |
+
2, 4
|
| 44 |
+
3, 4
|
| 45 |
+
4, 4
|
| 46 |
+
5, 4
|
| 47 |
+
6, 4
|
| 48 |
+
7, 4
|
| 49 |
+
0, 821
|
| 50 |
+
1, 821
|
| 51 |
+
2, 821
|
| 52 |
+
3, 821
|
| 53 |
+
4, 821
|
| 54 |
+
5, 821
|
| 55 |
+
6, 821
|
| 56 |
+
7, 821
|
| 57 |
+
0, 1927
|
| 58 |
+
1, 1927
|
| 59 |
+
2, 1927
|
| 60 |
+
3, 1927
|
| 61 |
+
4, 1927
|
| 62 |
+
5, 1927
|
| 63 |
+
6, 1927
|
| 64 |
+
7, 1927
|
| 65 |
+
0, 3783
|
| 66 |
+
1, 3767
|
| 67 |
+
2, 3769
|
| 68 |
+
3, 3777
|
| 69 |
+
4, 3795
|
| 70 |
+
5, 3779
|
| 71 |
+
6, 3769
|
| 72 |
+
7, 3765
|
| 73 |
+
0, 46859
|
| 74 |
+
1, 46859
|
| 75 |
+
2, 46859
|
| 76 |
+
3, 46859
|
| 77 |
+
4, 46859
|
| 78 |
+
5, 46859
|
| 79 |
+
6, 46859
|
| 80 |
+
7, 46859
|
| 81 |
+
0, 55553
|
| 82 |
+
1, 55553
|
| 83 |
+
2, 55553
|
| 84 |
+
3, 55553
|
| 85 |
+
4, 55553
|
| 86 |
+
5, 55553
|
| 87 |
+
6, 55553
|
| 88 |
+
7, 55553
|
| 89 |
+
0, 60691
|
| 90 |
+
1, 60693
|
| 91 |
+
2, 60691
|
| 92 |
+
3, 60691
|
| 93 |
+
4, 60691
|
| 94 |
+
5, 60691
|
| 95 |
+
6, 60691
|
| 96 |
+
7, 60691
|
| 97 |
+
0, 61267
|
| 98 |
+
1, 61269
|
| 99 |
+
2, 61267
|
| 100 |
+
3, 61267
|
| 101 |
+
4, 61267
|
| 102 |
+
5, 61267
|
| 103 |
+
6, 61267
|
| 104 |
+
7, 61267
|
| 105 |
+
0, 61267
|
| 106 |
+
1, 61269
|
| 107 |
+
2, 61267
|
| 108 |
+
3, 61267
|
| 109 |
+
4, 61267
|
| 110 |
+
5, 61267
|
| 111 |
+
6, 61267
|
| 112 |
+
7, 61267
|
| 113 |
+
0, 63677
|
| 114 |
+
1, 64467
|
| 115 |
+
2, 63851
|
| 116 |
+
3, 64465
|
| 117 |
+
4, 64465
|
| 118 |
+
5, 64423
|
| 119 |
+
6, 63679
|
| 120 |
+
7, 64465
|
| 121 |
+
0, 64465
|
| 122 |
+
1, 64467
|
| 123 |
+
2, 64465
|
| 124 |
+
3, 64465
|
| 125 |
+
4, 64465
|
| 126 |
+
5, 64465
|
| 127 |
+
6, 64465
|
| 128 |
+
7, 64465
|
| 129 |
+
0, 65963
|
| 130 |
+
1, 65965
|
| 131 |
+
2, 65963
|
| 132 |
+
3, 65963
|
| 133 |
+
4, 65963
|
| 134 |
+
5, 65963
|
| 135 |
+
6, 65963
|
| 136 |
+
7, 65963
|
| 137 |
+
0, 65963
|
| 138 |
+
1, 65965
|
| 139 |
+
2, 65963
|
| 140 |
+
3, 65963
|
| 141 |
+
4, 65963
|
| 142 |
+
5, 65963
|
| 143 |
+
6, 65963
|
| 144 |
+
7, 65963
|
| 145 |
+
0, 65969
|
| 146 |
+
1, 65969
|
| 147 |
+
2, 65969
|
| 148 |
+
3, 65969
|
| 149 |
+
4, 65969
|
| 150 |
+
5, 65969
|
| 151 |
+
6, 65969
|
| 152 |
+
7, 65969
|
| 153 |
+
0, 68223
|
| 154 |
+
1, 68257
|
| 155 |
+
2, 68253
|
| 156 |
+
3, 68345
|
| 157 |
+
4, 68283
|
| 158 |
+
5, 68271
|
| 159 |
+
6, 68275
|
| 160 |
+
7, 68315
|
| 161 |
+
0, 69183
|
| 162 |
+
1, 69199
|
| 163 |
+
2, 69199
|
| 164 |
+
3, 69213
|
| 165 |
+
4, 69239
|
| 166 |
+
5, 69175
|
| 167 |
+
6, 69191
|
| 168 |
+
7, 69187
|
| 169 |
+
0, 75761
|
| 170 |
+
1, 75727
|
| 171 |
+
2, 75751
|
| 172 |
+
3, 75785
|
| 173 |
+
4, 75801
|
| 174 |
+
5, 75763
|
| 175 |
+
6, 75819
|
| 176 |
+
7, 75791
|
| 177 |
+
0, 90291
|
| 178 |
+
1, 90297
|
| 179 |
+
2, 90291
|
| 180 |
+
3, 90347
|
| 181 |
+
4, 90347
|
| 182 |
+
5, 90347
|
| 183 |
+
6, 90375
|
| 184 |
+
7, 90347
|
| 185 |
+
0, 14406
|
| 186 |
+
1, 14406
|
| 187 |
+
2, 14406
|
| 188 |
+
3, 14790
|
| 189 |
+
4, 90347
|
| 190 |
+
5, 16326
|
| 191 |
+
6, 14406
|
| 192 |
+
7, 14406
|
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep2/run.env
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=ultraep
|
| 2 |
+
config=KIMI_K2_L2
|
| 3 |
+
family=tierB
|
| 4 |
+
source_model=Kimi-K2
|
| 5 |
+
standard_num_layers=61
|
| 6 |
+
base_rung=
|
| 7 |
+
repo=/workspace/Megatron-UltraEP
|
| 8 |
+
branch=ultraep-bracing-regions-through-kernels
|
| 9 |
+
commit=03d57ef7
|
| 10 |
+
num_layers=2
|
| 11 |
+
hidden=7168
|
| 12 |
+
ffn_hidden=18432
|
| 13 |
+
moe_ffn_hidden=2048
|
| 14 |
+
num_experts=384
|
| 15 |
+
topk=8
|
| 16 |
+
experts_per_rank=48
|
| 17 |
+
expert_params=44040192
|
| 18 |
+
heads=64
|
| 19 |
+
kv_groups=64
|
| 20 |
+
kv_channels=64
|
| 21 |
+
qk_layernorm=1
|
| 22 |
+
shared_ffn=2048
|
| 23 |
+
shared_expert_overlap=0
|
| 24 |
+
moe_token_dispatcher=flex
|
| 25 |
+
mla=1
|
| 26 |
+
q_lora_rank=1536
|
| 27 |
+
kv_lora_rank=512
|
| 28 |
+
qk_head_dim=128
|
| 29 |
+
qk_pos_emb_head_dim=64
|
| 30 |
+
v_head_dim=128
|
| 31 |
+
dense_first_layers=1
|
| 32 |
+
moe_layers=1
|
| 33 |
+
rotary_base=50000
|
| 34 |
+
rotary_percent=1.0
|
| 35 |
+
rotary_scaling=32
|
| 36 |
+
mscale=1.0
|
| 37 |
+
mscale_all_dim=1.0
|
| 38 |
+
qkv_bias=0
|
| 39 |
+
router_balancing=seq_aux_loss
|
| 40 |
+
aux_loss_coeff=0
|
| 41 |
+
router_pre_softmax=0
|
| 42 |
+
router_fusion=0
|
| 43 |
+
router_dtype=fp32
|
| 44 |
+
router_groups=1
|
| 45 |
+
router_group_topk=1
|
| 46 |
+
router_score_function=sigmoid
|
| 47 |
+
router_expert_bias=1
|
| 48 |
+
router_bias_update_rate=1e-3
|
| 49 |
+
router_topk_scaling_factor=2.827
|
| 50 |
+
shared_expert_gate=0
|
| 51 |
+
apply_layernorm_1p=0
|
| 52 |
+
attention_output_gate=0
|
| 53 |
+
linear_attention_type=
|
| 54 |
+
linear_attention_freq=
|
| 55 |
+
linear_conv_kernel_dim=
|
| 56 |
+
linear_key_head_dim=
|
| 57 |
+
linear_value_head_dim=
|
| 58 |
+
linear_num_key_heads=
|
| 59 |
+
linear_num_value_heads=
|
| 60 |
+
mtp_num_layers=0
|
| 61 |
+
mtp_loss_scaling_factor=0.1
|
| 62 |
+
no_weight_decay_cond_type=
|
| 63 |
+
max_position_embeddings=4096
|
| 64 |
+
init_method_std=0.02
|
| 65 |
+
device_initiated_grouped_gemm=1
|
| 66 |
+
norm_epsilon=1e-6
|
| 67 |
+
seq_length=4096
|
| 68 |
+
micro_batch_size=1
|
| 69 |
+
global_batch_size=8
|
| 70 |
+
num_microbatches=1
|
| 71 |
+
assignments_per_rank=32768
|
| 72 |
+
mean_tokens_per_expert=682
|
| 73 |
+
slot_policy=matched
|
| 74 |
+
slots_per_rank=48
|
| 75 |
+
capacity_mode=static
|
| 76 |
+
capacity=1.17578125
|
| 77 |
+
cuda_graph=1
|
| 78 |
+
probe_mode=0
|
| 79 |
+
async_grad_reduce=0
|
| 80 |
+
branch_role=timing
|
| 81 |
+
memory_dump_mode=none
|
| 82 |
+
memory_snapshot_path=
|
| 83 |
+
cuda_graph_warmup_steps=3
|
| 84 |
+
recompute=none
|
| 85 |
+
force_balance=0
|
| 86 |
+
train_iters=12
|
| 87 |
+
ep_size=8
|
| 88 |
+
echo_dispatch_overlap=1
|
| 89 |
+
echo_dispatch_recompute=0
|
| 90 |
+
ultraep_quota_min_tokens=85
|
| 91 |
+
ultraep_weight_sync_join=
|
| 92 |
+
profile_enable=0
|
| 93 |
+
kernel_trace=0
|
| 94 |
+
fp8_mode=mxfp8
|
| 95 |
+
fp8_recipe=mxfp8
|
| 96 |
+
training_seed=1234
|
| 97 |
+
learning_rate=1e-4
|
| 98 |
+
min_learning_rate=1.0e-5
|
| 99 |
+
lr_warmup_iters=5
|
| 100 |
+
state_dump_enable=0
|
| 101 |
+
state_dump_dir=
|
| 102 |
+
moonep_assert_no_pending_reduce=0
|
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep2/train.log
ADDED
|
@@ -0,0 +1,1072 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[07:48:32] WARN: ultraep-bracing-regions-through-kernels lacks --moe-router-fusion; KIMI_K2_L2 runs unfused (timing only, routing identical)
|
| 2 |
+
==============================================================
|
| 3 |
+
KIMI_K2_L2 / ultraep (slots/rank=48 policy=matched)
|
| 4 |
+
E=384 K=8 H'=2048 m=48
|
| 5 |
+
L=2 seq=4096 gbs=8 (1 microbatches)
|
| 6 |
+
cap_mode=static cap=1.17578125 cg=1 recompute=none
|
| 7 |
+
memory_dump=none snapshots=/workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep2/memory_snapshots/snapshot.pickle
|
| 8 |
+
repo=/workspace/Megatron-UltraEP @ 03d57ef7
|
| 9 |
+
==============================================================
|
| 10 |
+
W0921 07:48:33.484000 524467 torch/distributed/run.py:766]
|
| 11 |
+
W0921 07:48:33.484000 524467 torch/distributed/run.py:766] *****************************************
|
| 12 |
+
W0921 07:48:33.484000 524467 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 13 |
+
W0921 07:48:33.484000 524467 torch/distributed/run.py:766] *****************************************
|
| 14 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 15 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 16 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 17 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 18 |
+
using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1
|
| 19 |
+
WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer
|
| 20 |
+
Number of virtual stages per pipeline stage: None
|
| 21 |
+
accumulate and all-reduce gradients in fp32 for bfloat16 data type.
|
| 22 |
+
using torch.bfloat16 for parameters ...
|
| 23 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 24 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 25 |
+
------------------------ arguments ------------------------
|
| 26 |
+
account_for_embedding_in_pipeline_split ......... False
|
| 27 |
+
account_for_loss_in_pipeline_split .............. False
|
| 28 |
+
accumulate_allreduce_grads_in_fp32 .............. True
|
| 29 |
+
adam_beta1 ...................................... 0.9
|
| 30 |
+
adam_beta2 ...................................... 0.999
|
| 31 |
+
adam_eps ........................................ 1e-08
|
| 32 |
+
add_bias_linear ................................. False
|
| 33 |
+
add_position_embedding .......................... False
|
| 34 |
+
add_qkv_bias .................................... False
|
| 35 |
+
adlr_autoresume ................................. False
|
| 36 |
+
adlr_autoresume_interval ........................ 1000
|
| 37 |
+
align_grad_reduce ............................... True
|
| 38 |
+
align_param_gather .............................. False
|
| 39 |
+
app_tag_run_name ................................ None
|
| 40 |
+
app_tag_run_version ............................. 0.0.0
|
| 41 |
+
apply_layernorm_1p .............................. False
|
| 42 |
+
apply_query_key_layer_scaling ................... False
|
| 43 |
+
apply_residual_connection_post_layernorm ........ False
|
| 44 |
+
apply_rope_fusion ............................... False
|
| 45 |
+
async_save ...................................... None
|
| 46 |
+
async_tensor_model_parallel_allreduce ........... True
|
| 47 |
+
attention_backend ............................... AttnBackend.auto
|
| 48 |
+
attention_dropout ............................... 0.0
|
| 49 |
+
attention_softmax_in_fp32 ....................... False
|
| 50 |
+
auto_detect_ckpt_format ......................... False
|
| 51 |
+
barrier_with_L1_time ............................ True
|
| 52 |
+
bert_binary_head ................................ True
|
| 53 |
+
bert_embedder_type .............................. megatron
|
| 54 |
+
bert_load ....................................... None
|
| 55 |
+
bf16 ............................................ True
|
| 56 |
+
bias_dropout_fusion ............................. True
|
| 57 |
+
bias_gelu_fusion ................................ False
|
| 58 |
+
bias_swiglu_fusion .............................. True
|
| 59 |
+
biencoder_projection_dim ........................ 0
|
| 60 |
+
biencoder_shared_query_context_model ............ False
|
| 61 |
+
block_data_path ................................. None
|
| 62 |
+
calc_ft_timeouts ................................ False
|
| 63 |
+
calculate_per_token_loss ........................ False
|
| 64 |
+
check_for_large_grads ........................... False
|
| 65 |
+
check_for_nan_in_loss_and_grad .................. False
|
| 66 |
+
check_for_spiky_loss ............................ False
|
| 67 |
+
check_weight_hash_across_dp_replicas_interval ... None
|
| 68 |
+
ckpt_assume_constant_structure .................. False
|
| 69 |
+
ckpt_convert_format ............................. None
|
| 70 |
+
ckpt_convert_save ............................... None
|
| 71 |
+
ckpt_convert_update_legacy_dist_opt_format ...... False
|
| 72 |
+
ckpt_format ..................................... torch_dist
|
| 73 |
+
ckpt_fully_parallel_load ........................ False
|
| 74 |
+
ckpt_fully_parallel_save ........................ True
|
| 75 |
+
ckpt_fully_parallel_save_deprecated ............. False
|
| 76 |
+
ckpt_step ....................................... None
|
| 77 |
+
classes_fraction ................................ 1.0
|
| 78 |
+
clip_grad ....................................... 1.0
|
| 79 |
+
clone_scatter_output_in_embedding ............... True
|
| 80 |
+
config_logger_dir ...............................
|
| 81 |
+
consumed_train_samples .......................... 0
|
| 82 |
+
consumed_valid_samples .......................... 0
|
| 83 |
+
context_parallel_size ........................... 1
|
| 84 |
+
cp_comm_type .................................... ['p2p']
|
| 85 |
+
create_attention_mask_in_dataloader ............. True
|
| 86 |
+
cross_entropy_fusion_impl ....................... native
|
| 87 |
+
cross_entropy_loss_fusion ....................... False
|
| 88 |
+
cuda_graph_scope ................................ full_iteration
|
| 89 |
+
cuda_graph_warmup_steps ......................... 3
|
| 90 |
+
data_args_path .................................. None
|
| 91 |
+
data_cache_path ................................. None
|
| 92 |
+
data_parallel_random_init ....................... False
|
| 93 |
+
data_parallel_sharding_strategy ................. no_shard
|
| 94 |
+
data_parallel_size .............................. 8
|
| 95 |
+
data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document']
|
| 96 |
+
data_per_class_fraction ......................... 1.0
|
| 97 |
+
data_sharding ................................... True
|
| 98 |
+
dataloader_type ................................. single
|
| 99 |
+
ddp_average_in_collective ....................... False
|
| 100 |
+
ddp_bucket_size ................................. None
|
| 101 |
+
ddp_num_buckets ................................. None
|
| 102 |
+
ddp_pad_buckets_for_high_nccl_busbw ............. False
|
| 103 |
+
decoder_first_pipeline_num_layers ............... None
|
| 104 |
+
decoder_last_pipeline_num_layers ................ None
|
| 105 |
+
decoder_num_layers .............................. None
|
| 106 |
+
decoder_seq_length .............................. None
|
| 107 |
+
decoupled_lr .................................... None
|
| 108 |
+
decoupled_min_lr ................................ None
|
| 109 |
+
decrease_batch_size_if_needed ................... False
|
| 110 |
+
defer_embedding_wgrad_compute ................... False
|
| 111 |
+
delay_wgrad_compute ............................. False
|
| 112 |
+
deprecated_use_mcore_models ..................... True
|
| 113 |
+
deterministic_mode .............................. False
|
| 114 |
+
dino_bottleneck_size ............................ 256
|
| 115 |
+
dino_freeze_last_layer .......................... 1
|
| 116 |
+
dino_head_hidden_size ........................... 2048
|
| 117 |
+
dino_local_crops_number ......................... 10
|
| 118 |
+
dino_local_img_size ............................. 96
|
| 119 |
+
dino_norm_last_layer ............................ False
|
| 120 |
+
dino_teacher_temp ............................... 0.07
|
| 121 |
+
dino_warmup_teacher_temp ........................ 0.04
|
| 122 |
+
dino_warmup_teacher_temp_epochs ................. 30
|
| 123 |
+
disable_bf16_reduced_precision_matmul ........... False
|
| 124 |
+
disable_mamba_mem_eff_path ...................... False
|
| 125 |
+
disable_straggler_on_startup .................... False
|
| 126 |
+
dist_ckpt_format_deprecated ..................... None
|
| 127 |
+
dist_ckpt_strictness ............................ assume_ok_unexpected
|
| 128 |
+
distribute_saved_activations .................... False
|
| 129 |
+
distributed_backend ............................. nccl
|
| 130 |
+
distributed_timeout_minutes ..................... 10
|
| 131 |
+
embedding_init_method_std ....................... None
|
| 132 |
+
embedding_path .................................. None
|
| 133 |
+
empty_unused_memory_level ....................... 0
|
| 134 |
+
enable_cuda_graph ............................... True
|
| 135 |
+
enable_experimental ............................. False
|
| 136 |
+
enable_ft_package ............................... False
|
| 137 |
+
enable_gloo_process_groups ...................... True
|
| 138 |
+
enable_msc ...................................... True
|
| 139 |
+
enable_one_logger ............................... True
|
| 140 |
+
encoder_num_layers .............................. 2
|
| 141 |
+
encoder_seq_length .............................. 4096
|
| 142 |
+
end_weight_decay ................................ 0.1
|
| 143 |
+
eod_mask_loss ................................... False
|
| 144 |
+
error_injection_rate ............................ 0
|
| 145 |
+
error_injection_type ............................ transient_error
|
| 146 |
+
eval_interval ................................... 1000000
|
| 147 |
+
eval_iters ...................................... 0
|
| 148 |
+
evidence_data_path .............................. None
|
| 149 |
+
exit_duration_in_mins ........................... None
|
| 150 |
+
exit_interval ................................... None
|
| 151 |
+
exit_on_missing_checkpoint ...................... False
|
| 152 |
+
exit_signal_handler ............................. False
|
| 153 |
+
exp_avg_dtype ................................... torch.float32
|
| 154 |
+
exp_avg_sq_dtype ................................ torch.float32
|
| 155 |
+
expert_model_parallel_size ...................... 8
|
| 156 |
+
expert_tensor_parallel_size ..................... 1
|
| 157 |
+
export_force_local_attention .................... False
|
| 158 |
+
export_kd_cfg ................................... None
|
| 159 |
+
export_kd_teacher_ckpt_format ................... None
|
| 160 |
+
export_kd_teacher_load .......................... None
|
| 161 |
+
export_kv_cache_quant ........................... False
|
| 162 |
+
export_legacy_megatron .......................... False
|
| 163 |
+
export_model_type ............................... GPTModel
|
| 164 |
+
export_moe_apply_probs_on_input ................. False
|
| 165 |
+
export_qk_l2_norm ............................... False
|
| 166 |
+
export_quant_cfg ................................ None
|
| 167 |
+
export_real_quant_cfg ........................... None
|
| 168 |
+
export_te_mcore_model ........................... False
|
| 169 |
+
external_cuda_graph ............................. False
|
| 170 |
+
ffn_hidden_size ................................. 18432
|
| 171 |
+
finetune ........................................ False
|
| 172 |
+
finetune_data_split ............................. train
|
| 173 |
+
finetune_hf_dataset ............................. None
|
| 174 |
+
first_last_layers_bf16 .......................... False
|
| 175 |
+
flash_decode .................................... False
|
| 176 |
+
fp16 ............................................ False
|
| 177 |
+
fp16_lm_cross_entropy ........................... False
|
| 178 |
+
fp32_residual_connection ........................ False
|
| 179 |
+
fp8 ............................................. e4m3
|
| 180 |
+
fp8_amax_compute_algo ........................... most_recent
|
| 181 |
+
fp8_amax_history_len ............................ 1
|
| 182 |
+
fp8_interval .................................... 1
|
| 183 |
+
fp8_margin ...................................... 0
|
| 184 |
+
fp8_param_gather ................................ False
|
| 185 |
+
fp8_recipe ...................................... mxfp8
|
| 186 |
+
fp8_wgrad ....................................... True
|
| 187 |
+
fsdp_double_buffer .............................. False
|
| 188 |
+
global_batch_size ............................... 8
|
| 189 |
+
grad_reduce_in_bf16 ............................. False
|
| 190 |
+
gradient_accumulation_fusion .................... True
|
| 191 |
+
gradient_reduce_div_fusion ...................... True
|
| 192 |
+
group_query_attention ........................... False
|
| 193 |
+
head_lr_mult .................................... 1.0
|
| 194 |
+
heterogeneous_layers_config_encoded_json ........ None
|
| 195 |
+
heterogeneous_layers_config_path ................ None
|
| 196 |
+
hidden_dropout .................................. 0.0
|
| 197 |
+
hidden_size ..................................... 7168
|
| 198 |
+
hierarchical_context_parallel_sizes ............. None
|
| 199 |
+
high_priority_stream_groups ..................... []
|
| 200 |
+
hybrid_attention_ratio .......................... 0.0
|
| 201 |
+
hybrid_mlp_ratio ................................ 0.0
|
| 202 |
+
hybrid_override_pattern ......................... None
|
| 203 |
+
hysteresis ...................................... 2
|
| 204 |
+
ict_head_size ................................... None
|
| 205 |
+
ict_load ........................................ None
|
| 206 |
+
img_h ........................................... 224
|
| 207 |
+
img_w ........................................... 224
|
| 208 |
+
indexer_batch_size .............................. 128
|
| 209 |
+
indexer_log_interval ............................ 1000
|
| 210 |
+
inference_batch_times_seqlen_threshold .......... -1
|
| 211 |
+
inference_dynamic_batching ...................... False
|
| 212 |
+
inference_dynamic_batching_buffer_guaranteed_fraction 0.2
|
| 213 |
+
inference_dynamic_batching_buffer_overflow_factor None
|
| 214 |
+
inference_dynamic_batching_buffer_size_gb ....... 40.0
|
| 215 |
+
inference_dynamic_batching_chunk_size ........... 256
|
| 216 |
+
inference_dynamic_batching_max_requests_override None
|
| 217 |
+
inference_dynamic_batching_max_tokens_override .. None
|
| 218 |
+
inference_dynamic_batching_num_cuda_graphs ...... 16
|
| 219 |
+
inference_max_batch_size ........................ 8
|
| 220 |
+
inference_max_seq_length ........................ 2560
|
| 221 |
+
inference_rng_tracker ........................... False
|
| 222 |
+
init_method_std ................................. 0.02
|
| 223 |
+
init_method_xavier_uniform ...................... False
|
| 224 |
+
init_model_with_meta_device ..................... False
|
| 225 |
+
initial_loss_scale .............................. 4294967296
|
| 226 |
+
inprocess_active_world_size ..................... 8
|
| 227 |
+
inprocess_barrier_timeout ....................... 120
|
| 228 |
+
inprocess_completion_timeout .................... 120
|
| 229 |
+
inprocess_empty_cuda_cache ...................... False
|
| 230 |
+
inprocess_granularity ........................... node
|
| 231 |
+
inprocess_hard_timeout .......................... 90
|
| 232 |
+
inprocess_heartbeat_interval .................... 30
|
| 233 |
+
inprocess_heartbeat_timeout ..................... 60
|
| 234 |
+
inprocess_last_call_wait ........................ 1
|
| 235 |
+
inprocess_max_iterations ........................ None
|
| 236 |
+
inprocess_monitor_process_interval .............. 1.0
|
| 237 |
+
inprocess_monitor_thread_interval ............... 1.0
|
| 238 |
+
inprocess_progress_watchdog_interval ............ 1.0
|
| 239 |
+
inprocess_restart ............................... False
|
| 240 |
+
inprocess_soft_timeout .......................... 60
|
| 241 |
+
inprocess_termination_grace_time ................ 1
|
| 242 |
+
is_hybrid_model ................................. False
|
| 243 |
+
iter_per_epoch .................................. 1250
|
| 244 |
+
iterations_to_skip .............................. []
|
| 245 |
+
keep_fp8_transpose_cache_when_using_custom_fsdp . False
|
| 246 |
+
kitchen_config_file ............................. None
|
| 247 |
+
kitchen_recipe_number ........................... None
|
| 248 |
+
kv_channels ..................................... 64
|
| 249 |
+
kv_lora_rank .................................... 512
|
| 250 |
+
lazy_mpu_init ................................... None
|
| 251 |
+
load ............................................ None
|
| 252 |
+
load_main_params_from_ckpt ...................... None
|
| 253 |
+
load_model_opt_format ........................... False
|
| 254 |
+
local_rank ...................................... 0
|
| 255 |
+
log_energy ...................................... False
|
| 256 |
+
log_interval .................................... 1
|
| 257 |
+
log_loss_scale_to_tensorboard ................... True
|
| 258 |
+
log_memory_to_tensorboard ....................... False
|
| 259 |
+
log_num_zeros_in_grad ........................... False
|
| 260 |
+
log_params_norm ................................. False
|
| 261 |
+
log_progress .................................... False
|
| 262 |
+
log_straggler ................................... False
|
| 263 |
+
log_throughput .................................. True
|
| 264 |
+
log_timers_to_tensorboard ....................... False
|
| 265 |
+
log_validation_ppl_to_tensorboard ............... False
|
| 266 |
+
log_world_size_to_tensorboard ................... False
|
| 267 |
+
logging_level ................................... None
|
| 268 |
+
loss_scale ...................................... None
|
| 269 |
+
loss_scale_window ............................... 1000
|
| 270 |
+
lr .............................................. 0.0001
|
| 271 |
+
lr_decay_iters .................................. 8
|
| 272 |
+
lr_decay_samples ................................ None
|
| 273 |
+
lr_decay_style .................................. cosine
|
| 274 |
+
lr_warmup_fraction .............................. None
|
| 275 |
+
lr_warmup_init .................................. 0.0
|
| 276 |
+
lr_warmup_iters ................................. 5
|
| 277 |
+
lr_warmup_samples ............................... 0
|
| 278 |
+
lr_wsd_decay_iters .............................. None
|
| 279 |
+
lr_wsd_decay_samples ............................ None
|
| 280 |
+
lr_wsd_decay_style .............................. exponential
|
| 281 |
+
main_grads_dtype ................................ torch.float32
|
| 282 |
+
main_params_dtype ............................... torch.float32
|
| 283 |
+
make_vocab_size_divisible_by .................... 128
|
| 284 |
+
mamba_head_dim .................................. 64
|
| 285 |
+
mamba_num_groups ................................ 8
|
| 286 |
+
mamba_num_heads ................................. None
|
| 287 |
+
mamba_state_dim ................................. 128
|
| 288 |
+
manual_gc ....................................... False
|
| 289 |
+
manual_gc_eval .................................. True
|
| 290 |
+
manual_gc_interval .............................. 0
|
| 291 |
+
mask_factor ..................................... 1.0
|
| 292 |
+
mask_prob ....................................... 0.15
|
| 293 |
+
mask_type ....................................... random
|
| 294 |
+
masked_softmax_fusion ........................... False
|
| 295 |
+
max_position_embeddings ......................... 4096
|
| 296 |
+
max_tokens_to_oom ............................... 12000
|
| 297 |
+
memory_snapshot_path ............................ snapshot.pickle
|
| 298 |
+
merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt
|
| 299 |
+
micro_batch_size ................................ 1
|
| 300 |
+
microbatch_group_size_per_vp_stage .............. None
|
| 301 |
+
mid_level_dataset_surplus ....................... 0.005
|
| 302 |
+
min_loss_scale .................................. 1.0
|
| 303 |
+
min_lr .......................................... 1e-05
|
| 304 |
+
mlp_chunks_for_prefill .......................... 1
|
| 305 |
+
mmap_bin_files .................................. True
|
| 306 |
+
mock_data ....................................... False
|
| 307 |
+
moe_apply_probs_on_input ........................ False
|
| 308 |
+
moe_aux_loss_coeff .............................. 0.0
|
| 309 |
+
moe_capacity_monitor ............................ True
|
| 310 |
+
moe_deepep_num_sms .............................. 20
|
| 311 |
+
moe_echo_enable_random_offloading ............... False
|
| 312 |
+
moe_echo_expert_dispatch_overlap ................ False
|
| 313 |
+
moe_echo_expert_dispatcher_type ................. hybridep
|
| 314 |
+
moe_echo_recompute_expert_dispatch .............. False
|
| 315 |
+
moe_enable_deepep ............................... False
|
| 316 |
+
moe_enable_echo ................................. False
|
| 317 |
+
moe_enable_hybridep ............................. True
|
| 318 |
+
moe_enable_ultraep .............................. True
|
| 319 |
+
moe_expert_capacity_factor ...................... None
|
| 320 |
+
moe_extended_tp ................................. False
|
| 321 |
+
moe_ffn_hidden_size ............................. 2048
|
| 322 |
+
moe_grouped_gemm ................................ True
|
| 323 |
+
moe_hybridep_num_sms ............................ 16
|
| 324 |
+
moe_input_jitter_eps ............................ None
|
| 325 |
+
moe_layer_freq .................................. [0, 1]
|
| 326 |
+
moe_layer_recompute ............................. False
|
| 327 |
+
moe_num_echo_experts ............................ None
|
| 328 |
+
moe_num_redundant_experts_per_rank .............. 48
|
| 329 |
+
moe_pad_expert_input_to_capacity ................ False
|
| 330 |
+
moe_per_layer_logging ........................... False
|
| 331 |
+
moe_permute_fusion .............................. False
|
| 332 |
+
moe_received_token_capacity ..................... 1.17578125
|
| 333 |
+
moe_router_bias_update_rate ..................... 0.001
|
| 334 |
+
moe_router_dtype ................................ fp32
|
| 335 |
+
moe_router_enable_expert_bias ................... True
|
| 336 |
+
moe_router_force_load_balancing ................. False
|
| 337 |
+
moe_router_group_topk ........................... 1
|
| 338 |
+
moe_router_load_balancing_type .................. seq_aux_loss
|
| 339 |
+
moe_router_num_groups ........................... 1
|
| 340 |
+
moe_router_padding_for_fp8 ...................... False
|
| 341 |
+
moe_router_pre_softmax .......................... False
|
| 342 |
+
moe_router_score_function ....................... sigmoid
|
| 343 |
+
moe_router_topk ................................. 8
|
| 344 |
+
moe_router_topk_scaling_factor .................. 2.827
|
| 345 |
+
moe_shared_expert_intermediate_size ............. 2048
|
| 346 |
+
moe_shared_expert_overlap ....................... False
|
| 347 |
+
moe_token_dispatcher_type ....................... flex
|
| 348 |
+
moe_token_drop_policy ........................... probs
|
| 349 |
+
moe_ultraep_max_microbatches .................... None
|
| 350 |
+
moe_ultraep_overlap_weight_sync ................. True
|
| 351 |
+
moe_ultraep_weight_sync_join .................... before_dispatch
|
| 352 |
+
moe_upcycling_granularity ....................... 1
|
| 353 |
+
moe_use_device_initiated_grouped_gemm ........... True
|
| 354 |
+
moe_use_legacy_grouped_gemm ..................... False
|
| 355 |
+
moe_use_upcycling ............................... False
|
| 356 |
+
moe_z_loss_coeff ................................ None
|
| 357 |
+
mrope_section ................................... None
|
| 358 |
+
mscale .......................................... 1.0
|
| 359 |
+
mscale_all_dim .................................. 1.0
|
| 360 |
+
mtp_loss_scaling_factor ......................... 0.1
|
| 361 |
+
mtp_num_layers .................................. None
|
| 362 |
+
multi_latent_attention .......................... True
|
| 363 |
+
nccl_all_reduce_for_prefill ..................... False
|
| 364 |
+
nccl_communicator_config_path ................... None
|
| 365 |
+
nccl_ub ......................................... False
|
| 366 |
+
no_load_optim ................................... None
|
| 367 |
+
no_load_rng ..................................... None
|
| 368 |
+
no_persist_layer_norm ........................... False
|
| 369 |
+
no_rope_freq .................................... None
|
| 370 |
+
no_save_optim ................................... None
|
| 371 |
+
no_save_rng ..................................... None
|
| 372 |
+
non_persistent_ckpt_type ........................ None
|
| 373 |
+
non_persistent_global_ckpt_dir .................. None
|
| 374 |
+
non_persistent_local_ckpt_algo .................. fully_parallel
|
| 375 |
+
non_persistent_local_ckpt_dir ................... None
|
| 376 |
+
non_persistent_save_interval .................... None
|
| 377 |
+
norm_epsilon .................................... 1e-06
|
| 378 |
+
normalization ................................... RMSNorm
|
| 379 |
+
num_attention_heads ............................. 64
|
| 380 |
+
num_channels .................................... 3
|
| 381 |
+
num_classes ..................................... 1000
|
| 382 |
+
num_dataset_builder_threads ..................... 1
|
| 383 |
+
num_distributed_optimizer_instances ............. 1
|
| 384 |
+
num_experts ..................................... 384
|
| 385 |
+
num_layers ...................................... 2
|
| 386 |
+
num_layers_at_end_in_bf16 ....................... 1
|
| 387 |
+
num_layers_at_start_in_bf16 ..................... 1
|
| 388 |
+
num_layers_per_virtual_pipeline_stage ........... None
|
| 389 |
+
num_query_groups ................................ 1
|
| 390 |
+
num_virtual_stages_per_pipeline_rank ............ None
|
| 391 |
+
num_workers ..................................... 2
|
| 392 |
+
object_storage_cache_path ....................... None
|
| 393 |
+
one_logger_async ................................ False
|
| 394 |
+
one_logger_project .............................. megatron-lm
|
| 395 |
+
one_logger_run_name ............................. None
|
| 396 |
+
onnx_safe ....................................... None
|
| 397 |
+
openai_gelu ..................................... False
|
| 398 |
+
optimizer ....................................... adam
|
| 399 |
+
optimizer_cpu_offload ........................... False
|
| 400 |
+
optimizer_offload_fraction ...................... 1.0
|
| 401 |
+
output_bert_embeddings .......................... False
|
| 402 |
+
overlap_cpu_optimizer_d2h_h2d ................... False
|
| 403 |
+
overlap_grad_reduce ............................. True
|
| 404 |
+
overlap_moe_expert_parallel_comm ................ False
|
| 405 |
+
overlap_p2p_comm ................................ False
|
| 406 |
+
overlap_p2p_comm_warmup_flush ................... False
|
| 407 |
+
overlap_param_gather ............................ True
|
| 408 |
+
overlap_param_gather_with_optimizer_step ........ False
|
| 409 |
+
override_opt_param_scheduler .................... False
|
| 410 |
+
params_dtype .................................... torch.bfloat16
|
| 411 |
+
patch_dim ....................................... 16
|
| 412 |
+
per_split_data_args_path ........................ None
|
| 413 |
+
perform_initialization .......................... True
|
| 414 |
+
pin_cpu_grads ................................... True
|
| 415 |
+
pin_cpu_params .................................. True
|
| 416 |
+
pipeline_model_parallel_comm_backend ............ None
|
| 417 |
+
pipeline_model_parallel_layout .................. None
|
| 418 |
+
pipeline_model_parallel_size .................... 1
|
| 419 |
+
position_embedding_type ......................... rope
|
| 420 |
+
pretrained_checkpoint ........................... None
|
| 421 |
+
profile ......................................... False
|
| 422 |
+
profile_ranks ................................... [0]
|
| 423 |
+
profile_step_end ................................ 12
|
| 424 |
+
profile_step_start .............................. 10
|
| 425 |
+
q_lora_rank ..................................... 1536
|
| 426 |
+
qk_head_dim ..................................... 128
|
| 427 |
+
qk_l2_norm ...................................... False
|
| 428 |
+
qk_layernorm .................................... True
|
| 429 |
+
qk_pos_emb_head_dim ............................. 64
|
| 430 |
+
query_in_block_prob ............................. 0.1
|
| 431 |
+
rampup_batch_size ............................... None
|
| 432 |
+
rank ............................................ 0
|
| 433 |
+
recompute_granularity ........................... None
|
| 434 |
+
recompute_method ................................ None
|
| 435 |
+
recompute_modules ............................... None
|
| 436 |
+
recompute_num_layers ............................ None
|
| 437 |
+
record_memory_history ........................... False
|
| 438 |
+
relative_attention_max_distance ................. 128
|
| 439 |
+
relative_attention_num_buckets .................. 32
|
| 440 |
+
replication ..................................... False
|
| 441 |
+
replication_factor .............................. 2
|
| 442 |
+
replication_jump ................................ None
|
| 443 |
+
rerun_mode ...................................... validate_results
|
| 444 |
+
reset_attention_mask ............................ False
|
| 445 |
+
reset_position_ids .............................. False
|
| 446 |
+
result_rejected_tracker_filename ................ None
|
| 447 |
+
retriever_report_topk_accuracies ................ []
|
| 448 |
+
retriever_score_scaling ......................... False
|
| 449 |
+
retriever_seq_length ............................ 256
|
| 450 |
+
retro_add_retriever ............................. False
|
| 451 |
+
retro_attention_gate ............................ 1
|
| 452 |
+
retro_cyclic_train_iters ........................ None
|
| 453 |
+
retro_encoder_attention_dropout ................. 0.1
|
| 454 |
+
retro_encoder_hidden_dropout .................... 0.1
|
| 455 |
+
retro_encoder_layers ............................ 2
|
| 456 |
+
retro_num_neighbors ............................. 2
|
| 457 |
+
retro_num_retrieved_chunks ...................... 2
|
| 458 |
+
retro_project_dir ............................... None
|
| 459 |
+
retro_verify_neighbor_count ..................... True
|
| 460 |
+
reuse_grad_buf_for_mxfp8_param_ag ............... False
|
| 461 |
+
rope_scaling_factor ............................. 8.0
|
| 462 |
+
rope_type ....................................... None
|
| 463 |
+
rotary_base ..................................... 50000
|
| 464 |
+
rotary_interleaved .............................. False
|
| 465 |
+
rotary_percent .................................. 1.0
|
| 466 |
+
rotary_scaling_factor ........................... 32.0
|
| 467 |
+
rotary_seq_len_interpolation_factor ............. None
|
| 468 |
+
run_workload_inspector_server ................... False
|
| 469 |
+
sample_rate ..................................... 1.0
|
| 470 |
+
save ............................................ None
|
| 471 |
+
save_interval ................................... None
|
| 472 |
+
save_retain_interval ............................ None
|
| 473 |
+
scatter_gather_tensors_in_pipeline .............. True
|
| 474 |
+
seed ............................................ 1234
|
| 475 |
+
seq_length ...................................... 4096
|
| 476 |
+
sequence_parallel ............................... False
|
| 477 |
+
sft ............................................. False
|
| 478 |
+
sft_tokenizer_prompt_format ..................... nemotron-h-aligned
|
| 479 |
+
sgd_momentum .................................... 0.9
|
| 480 |
+
short_seq_prob .................................. 0.1
|
| 481 |
+
skip_train ...................................... False
|
| 482 |
+
skipped_train_samples ........................... 0
|
| 483 |
+
spec ............................................ None
|
| 484 |
+
split ........................................... 100,0,0
|
| 485 |
+
squared_relu .................................... False
|
| 486 |
+
start_weight_decay .............................. 0.1
|
| 487 |
+
straggler_ctrlr_port ............................ 65535
|
| 488 |
+
straggler_minmax_count .......................... 1
|
| 489 |
+
suggested_communication_unit_size ............... None
|
| 490 |
+
swiglu .......................................... True
|
| 491 |
+
swin_backbone_type .............................. tiny
|
| 492 |
+
symmetric_ar_type ............................... None
|
| 493 |
+
te_rng_tracker .................................. True
|
| 494 |
+
tensor_model_parallel_size ...................... 1
|
| 495 |
+
tensorboard_dir ................................. None
|
| 496 |
+
tensorboard_log_interval ........................ 1
|
| 497 |
+
tensorboard_queue_size .......................... 1000
|
| 498 |
+
test_data_path .................................. None
|
| 499 |
+
test_mode ....................................... False
|
| 500 |
+
tiktoken_num_special_tokens ..................... 1000
|
| 501 |
+
tiktoken_pattern ................................ None
|
| 502 |
+
tiktoken_special_tokens ......................... None
|
| 503 |
+
timing_log_level ................................ 0
|
| 504 |
+
timing_log_option ............................... minmax
|
| 505 |
+
titles_data_path ................................ None
|
| 506 |
+
tokenizer_model ................................. None
|
| 507 |
+
tokenizer_type .................................. GPT2BPETokenizer
|
| 508 |
+
torch_fsdp2_reshard_after_forward ............... True
|
| 509 |
+
tp_comm_bootstrap_backend ....................... nccl
|
| 510 |
+
tp_comm_bulk_dgrad .............................. True
|
| 511 |
+
tp_comm_bulk_wgrad .............................. True
|
| 512 |
+
tp_comm_overlap ................................. False
|
| 513 |
+
tp_comm_overlap_ag .............................. True
|
| 514 |
+
tp_comm_overlap_cfg ............................. None
|
| 515 |
+
tp_comm_overlap_rs .............................. True
|
| 516 |
+
tp_comm_overlap_rs_dgrad ........................ False
|
| 517 |
+
tp_comm_split_ag ................................ True
|
| 518 |
+
tp_comm_split_rs ................................ True
|
| 519 |
+
train_data_path ................................. None
|
| 520 |
+
train_iters ..................................... 12
|
| 521 |
+
train_samples ................................... None
|
| 522 |
+
train_sync_interval ............................. None
|
| 523 |
+
transformer_impl ................................ transformer_engine
|
| 524 |
+
transformer_pipeline_model_parallel_size ........ 1
|
| 525 |
+
untie_embeddings_and_output_weights ............. True
|
| 526 |
+
use_checkpoint_args ............................. False
|
| 527 |
+
use_checkpoint_opt_param_scheduler .............. False
|
| 528 |
+
use_cpu_initialization .......................... None
|
| 529 |
+
use_custom_fsdp ................................. False
|
| 530 |
+
use_dist_ckpt ................................... True
|
| 531 |
+
use_dist_ckpt_deprecated ........................ False
|
| 532 |
+
use_distributed_optimizer ....................... True
|
| 533 |
+
use_flash_attn .................................. False
|
| 534 |
+
use_legacy_models ............................... False
|
| 535 |
+
use_mp_args_from_checkpoint_args ................ False
|
| 536 |
+
use_one_sent_docs ............................... False
|
| 537 |
+
use_persistent_ckpt_worker ...................... False
|
| 538 |
+
use_precision_aware_optimizer ................... False
|
| 539 |
+
use_pytorch_profiler ............................ False
|
| 540 |
+
use_ring_exchange_p2p ........................... False
|
| 541 |
+
use_rope_scaling ................................ False
|
| 542 |
+
use_rotary_position_embeddings .................. False
|
| 543 |
+
use_sharp ....................................... False
|
| 544 |
+
use_tokenizer_model_from_checkpoint_args ........ True
|
| 545 |
+
use_torch_fsdp2 ................................. False
|
| 546 |
+
use_torch_optimizer_for_cpu_offload ............. False
|
| 547 |
+
use_tp_pp_dp_mapping ............................ False
|
| 548 |
+
v_head_dim ...................................... 128
|
| 549 |
+
valid_data_path ................................. None
|
| 550 |
+
variable_seq_lengths ............................ False
|
| 551 |
+
virtual_pipeline_model_parallel_size ............ None
|
| 552 |
+
vision_backbone_type ............................ vit
|
| 553 |
+
vision_pretraining .............................. False
|
| 554 |
+
vision_pretraining_type ......................... classify
|
| 555 |
+
vocab_extra_ids ................................. 0
|
| 556 |
+
vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json
|
| 557 |
+
vocab_size ...................................... None
|
| 558 |
+
wandb_exp_name ..................................
|
| 559 |
+
wandb_project ...................................
|
| 560 |
+
wandb_save_dir ..................................
|
| 561 |
+
weight_decay .................................... 0.1
|
| 562 |
+
weight_decay_incr_style ......................... constant
|
| 563 |
+
wgrad_deferral_limit ............................ 0
|
| 564 |
+
world_size ...................................... 8
|
| 565 |
+
yaml_cfg ........................................ None
|
| 566 |
+
-------------------- end of arguments ---------------------
|
| 567 |
+
INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1
|
| 568 |
+
> building GPT2BPETokenizer tokenizer ...
|
| 569 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 570 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 571 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 572 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 573 |
+
> padded vocab (size: 50257) with 47 dummy tokens (new size: 50304)
|
| 574 |
+
WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS
|
| 575 |
+
> initializing torch distributed ...
|
| 576 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 577 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 578 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 579 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 580 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 581 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 582 |
+
WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it
|
| 583 |
+
[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 584 |
+
[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 585 |
+
[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 586 |
+
[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 587 |
+
[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 588 |
+
[Gloo] Rank [Gloo] Rank 5 is connected to 77 is connected to [Gloo] Rank peer ranks. 76Expected number of connected peer ranks is : peer ranks. is connected to 7Expected number of connected peer ranks is : 7
|
| 589 |
+
7 peer ranks.
|
| 590 |
+
Expected number of connected peer ranks is : 7
|
| 591 |
+
[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 592 |
+
[Gloo] Rank 0 is connected to [Gloo] Rank 75 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. Expected number of connected peer ranks is :
|
| 593 |
+
7
|
| 594 |
+
[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 595 |
+
[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 596 |
+
[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 597 |
+
[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 598 |
+
[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 599 |
+
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
|
| 600 |
+
> initialized tensor model parallel with size 1
|
| 601 |
+
> initialized pipeline model parallel with size 1
|
| 602 |
+
> setting random seeds to 1234 ...
|
| 603 |
+
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
|
| 604 |
+
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
|
| 605 |
+
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
|
| 606 |
+
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
|
| 607 |
+
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
|
| 608 |
+
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
|
| 609 |
+
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
|
| 610 |
+
[rank4]:[W921 07:48:50.518278748 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 611 |
+
[rank3]:[W921 07:48:50.518509064 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 612 |
+
[rank2]:[W921 07:48:50.518511490 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 613 |
+
[rank1]:[W921 07:48:50.518613627 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 614 |
+
[rank5]:[W921 07:48:50.520159673 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 615 |
+
[rank6]:[W921 07:48:50.520258096 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 616 |
+
[rank7]:[W921 07:48:50.520356968 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 617 |
+
> compiling dataset index builder ...
|
| 618 |
+
make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
|
| 619 |
+
make: Nothing to be done for 'default'.
|
| 620 |
+
make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
|
| 621 |
+
>>> done with dataset index builder. Compilation time: 0.037 seconds
|
| 622 |
+
WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations.
|
| 623 |
+
> compiling and loading fused kernels ...
|
| 624 |
+
[rank0]:[W921 07:48:50.676485981 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 625 |
+
>>> done with compiling and loading fused kernels. Compilation time: 4.061 seconds
|
| 626 |
+
time to initialize megatron (seconds): 7.706
|
| 627 |
+
[after megatron is initialized] datetime: 2026-09-21 07:48:56
|
| 628 |
+
building GPT model ...
|
| 629 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 630 |
+
warnings.warn(
|
| 631 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 632 |
+
warnings.warn(
|
| 633 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 634 |
+
warnings.warn(
|
| 635 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 636 |
+
warnings.warn(
|
| 637 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 638 |
+
warnings.warn(
|
| 639 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 640 |
+
warnings.warn(
|
| 641 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 642 |
+
warnings.warn(
|
| 643 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 644 |
+
warnings.warn(
|
| 645 |
+
[INFO] Use MNNVL fabric: False
|
| 646 |
+
[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8.
|
| 647 |
+
NVSHMEM v3.7.2
|
| 648 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 649 |
+
|
| 650 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 651 |
+
|
| 652 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 653 |
+
|
| 654 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 655 |
+
|
| 656 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 657 |
+
|
| 658 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 659 |
+
|
| 660 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 661 |
+
|
| 662 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 663 |
+
|
| 664 |
+
UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive).
|
| 665 |
+
> number of parameters on (tensor, pipeline) model parallel rank (0, 0): 5594455040
|
| 666 |
+
INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False)
|
| 667 |
+
INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 9
|
| 668 |
+
Params for bucket 1 (360579072 elements, 360579072 padded size):
|
| 669 |
+
module.output_layer.weight
|
| 670 |
+
Params for bucket 2 (44047360 elements, 44047360 padded size):
|
| 671 |
+
module.decoder.final_layernorm.weight
|
| 672 |
+
module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight
|
| 673 |
+
module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight
|
| 674 |
+
Params for bucket 3 (45163520 elements, 45163520 padded size):
|
| 675 |
+
module.decoder.layers.1.pre_mlp_layernorm.weight
|
| 676 |
+
module.decoder.layers.1.self_attention.linear_kv_down_proj.weight
|
| 677 |
+
module.decoder.layers.1.self_attention.linear_q_down_proj.weight
|
| 678 |
+
module.decoder.layers.1.mlp.router.weight
|
| 679 |
+
module.decoder.layers.1.self_attention.linear_kv_up_proj.weight
|
| 680 |
+
module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight
|
| 681 |
+
module.decoder.layers.1.self_attention.linear_q_up_proj.weight
|
| 682 |
+
module.decoder.layers.1.self_attention.linear_q_up_proj.layer_norm_weight
|
| 683 |
+
Params for bucket 4 (58720256 elements, 58720256 padded size):
|
| 684 |
+
module.decoder.layers.1.self_attention.linear_proj.weight
|
| 685 |
+
Params for bucket 5 (132127744 elements, 132127744 padded size):
|
| 686 |
+
module.decoder.layers.1.input_layernorm.weight
|
| 687 |
+
module.decoder.layers.0.mlp.linear_fc2.weight
|
| 688 |
+
Params for bucket 6 (264241152 elements, 264241152 padded size):
|
| 689 |
+
module.decoder.layers.0.mlp.linear_fc1.weight
|
| 690 |
+
Params for bucket 7 (42411008 elements, 42411008 padded size):
|
| 691 |
+
module.decoder.layers.0.self_attention.linear_kv_down_proj.weight
|
| 692 |
+
module.decoder.layers.0.self_attention.linear_q_up_proj.layer_norm_weight
|
| 693 |
+
module.decoder.layers.0.self_attention.linear_q_down_proj.weight
|
| 694 |
+
module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight
|
| 695 |
+
module.decoder.layers.0.self_attention.linear_kv_up_proj.weight
|
| 696 |
+
module.decoder.layers.0.self_attention.linear_q_up_proj.weight
|
| 697 |
+
module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight
|
| 698 |
+
Params for bucket 8 (58720256 elements, 58720256 padded size):
|
| 699 |
+
module.decoder.layers.0.self_attention.linear_proj.weight
|
| 700 |
+
Params for bucket 9 (360586240 elements, 360586240 padded size):
|
| 701 |
+
module.decoder.layers.0.input_layernorm.weight
|
| 702 |
+
module.embedding.word_embeddings.weight
|
| 703 |
+
INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40
|
| 704 |
+
Params for bucket 1 (44040192 elements, 44040192 padded size):
|
| 705 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight47
|
| 706 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight45
|
| 707 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight46
|
| 708 |
+
Params for bucket 2 (44040192 elements, 44040192 padded size):
|
| 709 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight44
|
| 710 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight42
|
| 711 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight43
|
| 712 |
+
Params for bucket 3 (44040192 elements, 44040192 padded size):
|
| 713 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight41
|
| 714 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight39
|
| 715 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight40
|
| 716 |
+
Params for bucket 4 (44040192 elements, 44040192 padded size):
|
| 717 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight38
|
| 718 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight37
|
| 719 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight36
|
| 720 |
+
Params for bucket 5 (44040192 elements, 44040192 padded size):
|
| 721 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight35
|
| 722 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight34
|
| 723 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight33
|
| 724 |
+
Params for bucket 6 (44040192 elements, 44040192 padded size):
|
| 725 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight32
|
| 726 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight31
|
| 727 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight30
|
| 728 |
+
Params for bucket 7 (44040192 elements, 44040192 padded size):
|
| 729 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight29
|
| 730 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight28
|
| 731 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight27
|
| 732 |
+
Params for bucket 8 (44040192 elements, 44040192 padded size):
|
| 733 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight26
|
| 734 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight25
|
| 735 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight24
|
| 736 |
+
Params for bucket 9 (44040192 elements, 44040192 padded size):
|
| 737 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight23
|
| 738 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight21
|
| 739 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight22
|
| 740 |
+
Params for bucket 10 (44040192 elements, 44040192 padded size):
|
| 741 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight20
|
| 742 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight18
|
| 743 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight19
|
| 744 |
+
Params for bucket 11 (44040192 elements, 44040192 padded size):
|
| 745 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight17
|
| 746 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight15
|
| 747 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight16
|
| 748 |
+
Params for bucket 12 (44040192 elements, 44040192 padded size):
|
| 749 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight14
|
| 750 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight12
|
| 751 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight13
|
| 752 |
+
Params for bucket 13 (44040192 elements, 44040192 padded size):
|
| 753 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight11
|
| 754 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight9
|
| 755 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight10
|
| 756 |
+
Params for bucket 14 (44040192 elements, 44040192 padded size):
|
| 757 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight8
|
| 758 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight6
|
| 759 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight7
|
| 760 |
+
Params for bucket 15 (44040192 elements, 44040192 padded size):
|
| 761 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight5
|
| 762 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight3
|
| 763 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight4
|
| 764 |
+
Params for bucket 16 (44040192 elements, 44040192 padded size):
|
| 765 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight2
|
| 766 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight0
|
| 767 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight1
|
| 768 |
+
Params for bucket 17 (58720256 elements, 58720256 padded size):
|
| 769 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight47
|
| 770 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight46
|
| 771 |
+
Params for bucket 18 (58720256 elements, 58720256 padded size):
|
| 772 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight45
|
| 773 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight44
|
| 774 |
+
Params for bucket 19 (58720256 elements, 58720256 padded size):
|
| 775 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight43
|
| 776 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight42
|
| 777 |
+
Params for bucket 20 (58720256 elements, 58720256 padded size):
|
| 778 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight41
|
| 779 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight40
|
| 780 |
+
Params for bucket 21 (58720256 elements, 58720256 padded size):
|
| 781 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight39
|
| 782 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight38
|
| 783 |
+
Params for bucket 22 (58720256 elements, 58720256 padded size):
|
| 784 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight37
|
| 785 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight36
|
| 786 |
+
Params for bucket 23 (58720256 elements, 58720256 padded size):
|
| 787 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight35
|
| 788 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight34
|
| 789 |
+
Params for bucket 24 (58720256 elements, 58720256 padded size):
|
| 790 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight33
|
| 791 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight32
|
| 792 |
+
Params for bucket 25 (58720256 elements, 58720256 padded size):
|
| 793 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight31
|
| 794 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight30
|
| 795 |
+
Params for bucket 26 (58720256 elements, 58720256 padded size):
|
| 796 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight29
|
| 797 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight28
|
| 798 |
+
Params for bucket 27 (58720256 elements, 58720256 padded size):
|
| 799 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight27
|
| 800 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight26
|
| 801 |
+
Params for bucket 28 (58720256 elements, 58720256 padded size):
|
| 802 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight25
|
| 803 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight24
|
| 804 |
+
Params for bucket 29 (58720256 elements, 58720256 padded size):
|
| 805 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight23
|
| 806 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight22
|
| 807 |
+
Params for bucket 30 (58720256 elements, 58720256 padded size):
|
| 808 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight21
|
| 809 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight20
|
| 810 |
+
Params for bucket 31 (58720256 elements, 58720256 padded size):
|
| 811 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight19
|
| 812 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight18
|
| 813 |
+
Params for bucket 32 (58720256 elements, 58720256 padded size):
|
| 814 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight17
|
| 815 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight16
|
| 816 |
+
Params for bucket 33 (58720256 elements, 58720256 padded size):
|
| 817 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight15
|
| 818 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight14
|
| 819 |
+
Params for bucket 34 (58720256 elements, 58720256 padded size):
|
| 820 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight13
|
| 821 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight12
|
| 822 |
+
Params for bucket 35 (58720256 elements, 58720256 padded size):
|
| 823 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight11
|
| 824 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight10
|
| 825 |
+
Params for bucket 36 (58720256 elements, 58720256 padded size):
|
| 826 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight9
|
| 827 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight8
|
| 828 |
+
Params for bucket 37 (58720256 elements, 58720256 padded size):
|
| 829 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight7
|
| 830 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight6
|
| 831 |
+
Params for bucket 38 (58720256 elements, 58720256 padded size):
|
| 832 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight5
|
| 833 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight4
|
| 834 |
+
Params for bucket 39 (58720256 elements, 58720256 padded size):
|
| 835 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight3
|
| 836 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight2
|
| 837 |
+
Params for bucket 40 (58720256 elements, 58720256 padded size):
|
| 838 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight1
|
| 839 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight0
|
| 840 |
+
INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=<megatron.core.timers.Timers object at 0x71d5f8ed6a50>, config_logger_dir='')
|
| 841 |
+
INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine
|
| 842 |
+
[after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-21 07:49:00
|
| 843 |
+
> building train, validation, and test datasets ...
|
| 844 |
+
> datasets target sizes (minimum size):
|
| 845 |
+
train: 96
|
| 846 |
+
validation: 0
|
| 847 |
+
test: 0
|
| 848 |
+
INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None]
|
| 849 |
+
> building train, validation, and test datasets for GPT ...
|
| 850 |
+
INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=<megatron.training.tokenizer.tokenizer._GPT2BPETokenizer object at 0x71d5f93d2ae0>, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None)
|
| 851 |
+
INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx
|
| 852 |
+
INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths
|
| 853 |
+
INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers
|
| 854 |
+
INFO:megatron.core.datasets.indexed_dataset: Extract the document indices
|
| 855 |
+
INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029
|
| 856 |
+
INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350
|
| 857 |
+
INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices
|
| 858 |
+
INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy
|
| 859 |
+
INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy
|
| 860 |
+
INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy
|
| 861 |
+
INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073
|
| 862 |
+
> finished creating GPT datasets ...
|
| 863 |
+
[after dataloaders are built] datetime: 2026-09-21 07:49:00
|
| 864 |
+
done with setup ...
|
| 865 |
+
(min, max) time across ranks (ms):
|
| 866 |
+
model-and-optimizer-setup ......................: (3818.24, 3822.76)
|
| 867 |
+
train/valid/test-data-iterators-setup ..........: (72.48, 94.09)
|
| 868 |
+
training ...
|
| 869 |
+
Overwriting rerun_state_machine.current_iteration from -1 to 0...
|
| 870 |
+
[before the start of training step] datetime: 2026-09-21 07:49:00
|
| 871 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 872 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 873 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 874 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 875 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 876 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 877 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 878 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 879 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 880 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 881 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 882 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 883 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 884 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 885 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 886 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 887 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 888 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 889 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 890 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 891 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 892 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 893 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 894 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 895 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 896 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 897 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 898 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 899 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 900 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 901 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 902 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 903 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 904 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 905 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 906 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 907 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 908 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 909 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 910 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 911 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 912 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 913 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 914 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 915 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 916 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 917 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 918 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 919 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 920 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 921 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 922 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 923 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 924 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 925 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 926 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 927 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 928 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 929 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 930 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 931 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 932 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 933 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 934 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 935 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 936 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 937 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 938 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 939 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 940 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 941 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 942 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 943 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 944 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 945 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 946 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 947 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 948 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 949 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 950 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 951 |
+
[2026-09-21 07:49:29] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 28413.5 | throughput per GPU (TFLOP/s/GPU): 1.2 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.227924E+01 | loss scale: 1.0 | grad norm: 12.617 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 |
|
| 952 |
+
Number of parameters in transformer block in billions: 17.55
|
| 953 |
+
Number of parameters in embedding layers in billions: 0.72
|
| 954 |
+
Total number of parameters in billions: 18.28
|
| 955 |
+
Number of parameters in most loaded shard in billions: 18.2753
|
| 956 |
+
Theoretical memory footprints: weight and optimizer=130715.23 MB
|
| 957 |
+
[Rank 0] (after 1 iterations) memory (MB) | allocated: 49136.10400390625 | max allocated: 49136.1044921875 | reserved: 54394.0 | max reserved: 54394.0
|
| 958 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 959 |
+
warnings.warn(
|
| 960 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 961 |
+
warnings.warn(
|
| 962 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 963 |
+
warnings.warn(
|
| 964 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 965 |
+
warnings.warn(
|
| 966 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 967 |
+
warnings.warn(
|
| 968 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 969 |
+
warnings.warn(
|
| 970 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 971 |
+
warnings.warn(
|
| 972 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 973 |
+
warnings.warn(
|
| 974 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 975 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 976 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 977 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 978 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 979 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 980 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 981 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 982 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 983 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 984 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 985 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 986 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 987 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 988 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 989 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 990 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 991 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 992 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 993 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 994 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 995 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 996 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 997 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 998 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 999 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1000 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 1001 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 1002 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1003 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1004 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 1005 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 1006 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1007 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1008 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1009 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1010 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1011 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1012 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1013 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1014 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1015 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1016 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1017 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1018 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1019 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1020 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1021 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1022 |
+
[2026-09-21 07:49:29] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 166.7 | throughput per GPU (TFLOP/s/GPU): 212.2 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.227088E+01 | loss scale: 1.0 | grad norm: 12.534 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 |
|
| 1023 |
+
[2026-09-21 07:49:29] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 111.9 | throughput per GPU (TFLOP/s/GPU): 316.1 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.088328E+01 | loss scale: 1.0 | grad norm: 11.023 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 |
|
| 1024 |
+
Capture CUDA graph for training!!!
|
| 1025 |
+
CUDA graph capture done!!!
|
| 1026 |
+
[2026-09-21 07:49:30] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 1272.5 | throughput per GPU (TFLOP/s/GPU): 27.8 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.448181E+01 | loss scale: 1.0 | grad norm: 72.911 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36608 | min safe capacity: 1.1172 |
|
| 1027 |
+
[2026-09-21 07:49:30] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 110.5 | throughput per GPU (TFLOP/s/GPU): 320.1 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.808459E+01 | loss scale: 1.0 | grad norm: 108.268 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1028 |
+
[2026-09-21 07:49:30] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 111.5 | throughput per GPU (TFLOP/s/GPU): 317.3 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.491727E+01 | loss scale: 1.0 | grad norm: 83.473 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1029 |
+
[2026-09-21 07:49:31] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 114.8 | throughput per GPU (TFLOP/s/GPU): 308.1 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.143662E+01 | loss scale: 1.0 | grad norm: 26.627 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1030 |
+
[2026-09-21 07:49:31] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 108.9 | throughput per GPU (TFLOP/s/GPU): 324.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.166943E+01 | loss scale: 1.0 | grad norm: 12.800 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1031 |
+
[2026-09-21 07:49:31] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 108.9 | throughput per GPU (TFLOP/s/GPU): 324.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.131635E+01 | loss scale: 1.0 | grad norm: 10.441 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1032 |
+
[2026-09-21 07:49:31] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 109.1 | throughput per GPU (TFLOP/s/GPU): 324.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.096943E+01 | loss scale: 1.0 | grad norm: 9.147 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1033 |
+
[2026-09-21 07:49:31] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 108.6 | throughput per GPU (TFLOP/s/GPU): 325.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.073911E+01 | loss scale: 1.0 | grad norm: 9.755 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1034 |
+
[2026-09-21 07:49:31] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 108.9 | throughput per GPU (TFLOP/s/GPU): 324.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.048286E+01 | loss scale: 1.0 | grad norm: 8.843 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1035 |
+
[after training is done] datetime: 2026-09-21 07:49:31
|
| 1036 |
+
W0921 07:49:32.152000 524467 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers
|
| 1037 |
+
W0921 07:49:32.152000 524467 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 524570 closing signal SIGTERM
|
| 1038 |
+
W0921 07:49:32.153000 524467 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 524571 closing signal SIGTERM
|
| 1039 |
+
W0921 07:49:32.154000 524467 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 524572 closing signal SIGTERM
|
| 1040 |
+
W0921 07:49:32.155000 524467 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 524573 closing signal SIGTERM
|
| 1041 |
+
W0921 07:49:32.156000 524467 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 524574 closing signal SIGTERM
|
| 1042 |
+
W0921 07:49:32.158000 524467 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 524575 closing signal SIGTERM
|
| 1043 |
+
W0921 07:49:32.159000 524467 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 524576 closing signal SIGTERM
|
| 1044 |
+
W0921 07:49:32.160000 524467 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 524577 closing signal SIGTERM
|
| 1045 |
+
Traceback (most recent call last):
|
| 1046 |
+
File "/usr/local/bin/torchrun", line 8, in <module>
|
| 1047 |
+
sys.exit(main())
|
| 1048 |
+
^^^^^^
|
| 1049 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
|
| 1050 |
+
return f(*args, **kwargs)
|
| 1051 |
+
^^^^^^^^^^^^^^^^^^
|
| 1052 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main
|
| 1053 |
+
run(args)
|
| 1054 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run
|
| 1055 |
+
elastic_launch(
|
| 1056 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__
|
| 1057 |
+
return launch_agent(self._config, self._entrypoint, list(args))
|
| 1058 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 1059 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent
|
| 1060 |
+
result = agent.run()
|
| 1061 |
+
^^^^^^^^^^^
|
| 1062 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
| 1063 |
+
result = f(*args, **kwargs)
|
| 1064 |
+
^^^^^^^^^^^^^^^^^^
|
| 1065 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run
|
| 1066 |
+
result = self._invoke_run(role)
|
| 1067 |
+
^^^^^^^^^^^^^^^^^^^^^^
|
| 1068 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run
|
| 1069 |
+
time.sleep(monitor_interval)
|
| 1070 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler
|
| 1071 |
+
raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
|
| 1072 |
+
torch.distributed.elastic.multiprocessing.api.SignalException: Process 524467 got signal: 15
|
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep3/mem_samples.csv
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
0, 0
|
| 2 |
+
1, 0
|
| 3 |
+
2, 0
|
| 4 |
+
3, 0
|
| 5 |
+
4, 0
|
| 6 |
+
5, 0
|
| 7 |
+
6, 0
|
| 8 |
+
7, 0
|
| 9 |
+
0, 0
|
| 10 |
+
1, 0
|
| 11 |
+
2, 0
|
| 12 |
+
3, 0
|
| 13 |
+
4, 0
|
| 14 |
+
5, 0
|
| 15 |
+
6, 0
|
| 16 |
+
7, 0
|
| 17 |
+
0, 4
|
| 18 |
+
1, 1
|
| 19 |
+
2, 1
|
| 20 |
+
3, 1
|
| 21 |
+
4, 1
|
| 22 |
+
5, 1
|
| 23 |
+
6, 1
|
| 24 |
+
7, 1
|
| 25 |
+
0, 4
|
| 26 |
+
1, 4
|
| 27 |
+
2, 4
|
| 28 |
+
3, 4
|
| 29 |
+
4, 4
|
| 30 |
+
5, 1
|
| 31 |
+
6, 1
|
| 32 |
+
7, 1
|
| 33 |
+
0, 4
|
| 34 |
+
1, 4
|
| 35 |
+
2, 4
|
| 36 |
+
3, 4
|
| 37 |
+
4, 4
|
| 38 |
+
5, 4
|
| 39 |
+
6, 4
|
| 40 |
+
7, 4
|
| 41 |
+
0, 4
|
| 42 |
+
1, 4
|
| 43 |
+
2, 4
|
| 44 |
+
3, 4
|
| 45 |
+
4, 4
|
| 46 |
+
5, 4
|
| 47 |
+
6, 4
|
| 48 |
+
7, 35
|
| 49 |
+
0, 821
|
| 50 |
+
1, 821
|
| 51 |
+
2, 821
|
| 52 |
+
3, 821
|
| 53 |
+
4, 821
|
| 54 |
+
5, 821
|
| 55 |
+
6, 821
|
| 56 |
+
7, 821
|
| 57 |
+
0, 1783
|
| 58 |
+
1, 1783
|
| 59 |
+
2, 1783
|
| 60 |
+
3, 1783
|
| 61 |
+
4, 1783
|
| 62 |
+
5, 1783
|
| 63 |
+
6, 1783
|
| 64 |
+
7, 1783
|
| 65 |
+
0, 3437
|
| 66 |
+
1, 3493
|
| 67 |
+
2, 2681
|
| 68 |
+
3, 2681
|
| 69 |
+
4, 2681
|
| 70 |
+
5, 3493
|
| 71 |
+
6, 2681
|
| 72 |
+
7, 3437
|
| 73 |
+
0, 4289
|
| 74 |
+
1, 4289
|
| 75 |
+
2, 4289
|
| 76 |
+
3, 4289
|
| 77 |
+
4, 4289
|
| 78 |
+
5, 4289
|
| 79 |
+
6, 4289
|
| 80 |
+
7, 4289
|
| 81 |
+
0, 50889
|
| 82 |
+
1, 50831
|
| 83 |
+
2, 50857
|
| 84 |
+
3, 50859
|
| 85 |
+
4, 50863
|
| 86 |
+
5, 50903
|
| 87 |
+
6, 50897
|
| 88 |
+
7, 50915
|
| 89 |
+
0, 60211
|
| 90 |
+
1, 60277
|
| 91 |
+
2, 60233
|
| 92 |
+
3, 60227
|
| 93 |
+
4, 60239
|
| 94 |
+
5, 60233
|
| 95 |
+
6, 60241
|
| 96 |
+
7, 60243
|
| 97 |
+
0, 61267
|
| 98 |
+
1, 61269
|
| 99 |
+
2, 61267
|
| 100 |
+
3, 61267
|
| 101 |
+
4, 61267
|
| 102 |
+
5, 61267
|
| 103 |
+
6, 61267
|
| 104 |
+
7, 61267
|
| 105 |
+
0, 61267
|
| 106 |
+
1, 61269
|
| 107 |
+
2, 61267
|
| 108 |
+
3, 61267
|
| 109 |
+
4, 61267
|
| 110 |
+
5, 61267
|
| 111 |
+
6, 61267
|
| 112 |
+
7, 61267
|
| 113 |
+
0, 62923
|
| 114 |
+
1, 62925
|
| 115 |
+
2, 62823
|
| 116 |
+
3, 62895
|
| 117 |
+
4, 62981
|
| 118 |
+
5, 63097
|
| 119 |
+
6, 62923
|
| 120 |
+
7, 62925
|
| 121 |
+
0, 64465
|
| 122 |
+
1, 64467
|
| 123 |
+
2, 64465
|
| 124 |
+
3, 64465
|
| 125 |
+
4, 64465
|
| 126 |
+
5, 64465
|
| 127 |
+
6, 64465
|
| 128 |
+
7, 64465
|
| 129 |
+
0, 65963
|
| 130 |
+
1, 65965
|
| 131 |
+
2, 65963
|
| 132 |
+
3, 65963
|
| 133 |
+
4, 65963
|
| 134 |
+
5, 65963
|
| 135 |
+
6, 65963
|
| 136 |
+
7, 65963
|
| 137 |
+
0, 65963
|
| 138 |
+
1, 65965
|
| 139 |
+
2, 65963
|
| 140 |
+
3, 65963
|
| 141 |
+
4, 65963
|
| 142 |
+
5, 65963
|
| 143 |
+
6, 65963
|
| 144 |
+
7, 65963
|
| 145 |
+
0, 65969
|
| 146 |
+
1, 65969
|
| 147 |
+
2, 65969
|
| 148 |
+
3, 65969
|
| 149 |
+
4, 65969
|
| 150 |
+
5, 65969
|
| 151 |
+
6, 65969
|
| 152 |
+
7, 65969
|
| 153 |
+
0, 68243
|
| 154 |
+
1, 68215
|
| 155 |
+
2, 68215
|
| 156 |
+
3, 68245
|
| 157 |
+
4, 68263
|
| 158 |
+
5, 68267
|
| 159 |
+
6, 68289
|
| 160 |
+
7, 68279
|
| 161 |
+
0, 69055
|
| 162 |
+
1, 69055
|
| 163 |
+
2, 69055
|
| 164 |
+
3, 69055
|
| 165 |
+
4, 69055
|
| 166 |
+
5, 69055
|
| 167 |
+
6, 69055
|
| 168 |
+
7, 69055
|
| 169 |
+
0, 75835
|
| 170 |
+
1, 75843
|
| 171 |
+
2, 75809
|
| 172 |
+
3, 75865
|
| 173 |
+
4, 75851
|
| 174 |
+
5, 75833
|
| 175 |
+
6, 75893
|
| 176 |
+
7, 75841
|
| 177 |
+
0, 90291
|
| 178 |
+
1, 90297
|
| 179 |
+
2, 90291
|
| 180 |
+
3, 90347
|
| 181 |
+
4, 90347
|
| 182 |
+
5, 90347
|
| 183 |
+
6, 90375
|
| 184 |
+
7, 90347
|
| 185 |
+
0, 14403
|
| 186 |
+
1, 14406
|
| 187 |
+
2, 14406
|
| 188 |
+
3, 14406
|
| 189 |
+
4, 14406
|
| 190 |
+
5, 14790
|
| 191 |
+
6, 90375
|
| 192 |
+
7, 16326
|
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep3/run.env
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=ultraep
|
| 2 |
+
config=KIMI_K2_L2
|
| 3 |
+
family=tierB
|
| 4 |
+
source_model=Kimi-K2
|
| 5 |
+
standard_num_layers=61
|
| 6 |
+
base_rung=
|
| 7 |
+
repo=/workspace/Megatron-UltraEP
|
| 8 |
+
branch=ultraep-bracing-regions-through-kernels
|
| 9 |
+
commit=03d57ef7
|
| 10 |
+
num_layers=2
|
| 11 |
+
hidden=7168
|
| 12 |
+
ffn_hidden=18432
|
| 13 |
+
moe_ffn_hidden=2048
|
| 14 |
+
num_experts=384
|
| 15 |
+
topk=8
|
| 16 |
+
experts_per_rank=48
|
| 17 |
+
expert_params=44040192
|
| 18 |
+
heads=64
|
| 19 |
+
kv_groups=64
|
| 20 |
+
kv_channels=64
|
| 21 |
+
qk_layernorm=1
|
| 22 |
+
shared_ffn=2048
|
| 23 |
+
shared_expert_overlap=0
|
| 24 |
+
moe_token_dispatcher=flex
|
| 25 |
+
mla=1
|
| 26 |
+
q_lora_rank=1536
|
| 27 |
+
kv_lora_rank=512
|
| 28 |
+
qk_head_dim=128
|
| 29 |
+
qk_pos_emb_head_dim=64
|
| 30 |
+
v_head_dim=128
|
| 31 |
+
dense_first_layers=1
|
| 32 |
+
moe_layers=1
|
| 33 |
+
rotary_base=50000
|
| 34 |
+
rotary_percent=1.0
|
| 35 |
+
rotary_scaling=32
|
| 36 |
+
mscale=1.0
|
| 37 |
+
mscale_all_dim=1.0
|
| 38 |
+
qkv_bias=0
|
| 39 |
+
router_balancing=seq_aux_loss
|
| 40 |
+
aux_loss_coeff=0
|
| 41 |
+
router_pre_softmax=0
|
| 42 |
+
router_fusion=0
|
| 43 |
+
router_dtype=fp32
|
| 44 |
+
router_groups=1
|
| 45 |
+
router_group_topk=1
|
| 46 |
+
router_score_function=sigmoid
|
| 47 |
+
router_expert_bias=1
|
| 48 |
+
router_bias_update_rate=1e-3
|
| 49 |
+
router_topk_scaling_factor=2.827
|
| 50 |
+
shared_expert_gate=0
|
| 51 |
+
apply_layernorm_1p=0
|
| 52 |
+
attention_output_gate=0
|
| 53 |
+
linear_attention_type=
|
| 54 |
+
linear_attention_freq=
|
| 55 |
+
linear_conv_kernel_dim=
|
| 56 |
+
linear_key_head_dim=
|
| 57 |
+
linear_value_head_dim=
|
| 58 |
+
linear_num_key_heads=
|
| 59 |
+
linear_num_value_heads=
|
| 60 |
+
mtp_num_layers=0
|
| 61 |
+
mtp_loss_scaling_factor=0.1
|
| 62 |
+
no_weight_decay_cond_type=
|
| 63 |
+
max_position_embeddings=4096
|
| 64 |
+
init_method_std=0.02
|
| 65 |
+
device_initiated_grouped_gemm=1
|
| 66 |
+
norm_epsilon=1e-6
|
| 67 |
+
seq_length=4096
|
| 68 |
+
micro_batch_size=1
|
| 69 |
+
global_batch_size=8
|
| 70 |
+
num_microbatches=1
|
| 71 |
+
assignments_per_rank=32768
|
| 72 |
+
mean_tokens_per_expert=682
|
| 73 |
+
slot_policy=matched
|
| 74 |
+
slots_per_rank=48
|
| 75 |
+
capacity_mode=static
|
| 76 |
+
capacity=1.17578125
|
| 77 |
+
cuda_graph=1
|
| 78 |
+
probe_mode=0
|
| 79 |
+
async_grad_reduce=0
|
| 80 |
+
branch_role=timing
|
| 81 |
+
memory_dump_mode=none
|
| 82 |
+
memory_snapshot_path=
|
| 83 |
+
cuda_graph_warmup_steps=3
|
| 84 |
+
recompute=none
|
| 85 |
+
force_balance=0
|
| 86 |
+
train_iters=12
|
| 87 |
+
ep_size=8
|
| 88 |
+
echo_dispatch_overlap=1
|
| 89 |
+
echo_dispatch_recompute=0
|
| 90 |
+
ultraep_quota_min_tokens=85
|
| 91 |
+
ultraep_weight_sync_join=
|
| 92 |
+
profile_enable=0
|
| 93 |
+
kernel_trace=0
|
| 94 |
+
fp8_mode=mxfp8
|
| 95 |
+
fp8_recipe=mxfp8
|
| 96 |
+
training_seed=1234
|
| 97 |
+
learning_rate=1e-4
|
| 98 |
+
min_learning_rate=1.0e-5
|
| 99 |
+
lr_warmup_iters=5
|
| 100 |
+
state_dump_enable=0
|
| 101 |
+
state_dump_dir=
|
| 102 |
+
moonep_assert_no_pending_reduce=0
|
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep3/train.log
ADDED
|
@@ -0,0 +1,1072 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[07:49:37] WARN: ultraep-bracing-regions-through-kernels lacks --moe-router-fusion; KIMI_K2_L2 runs unfused (timing only, routing identical)
|
| 2 |
+
==============================================================
|
| 3 |
+
KIMI_K2_L2 / ultraep (slots/rank=48 policy=matched)
|
| 4 |
+
E=384 K=8 H'=2048 m=48
|
| 5 |
+
L=2 seq=4096 gbs=8 (1 microbatches)
|
| 6 |
+
cap_mode=static cap=1.17578125 cg=1 recompute=none
|
| 7 |
+
memory_dump=none snapshots=/workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep3/memory_snapshots/snapshot.pickle
|
| 8 |
+
repo=/workspace/Megatron-UltraEP @ 03d57ef7
|
| 9 |
+
==============================================================
|
| 10 |
+
W0921 07:49:38.672000 527457 torch/distributed/run.py:766]
|
| 11 |
+
W0921 07:49:38.672000 527457 torch/distributed/run.py:766] *****************************************
|
| 12 |
+
W0921 07:49:38.672000 527457 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 13 |
+
W0921 07:49:38.672000 527457 torch/distributed/run.py:766] *****************************************
|
| 14 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 15 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 16 |
+
WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it
|
| 17 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 18 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 19 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 20 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 21 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 22 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 23 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 24 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 25 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 26 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 27 |
+
using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1
|
| 28 |
+
WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer
|
| 29 |
+
Number of virtual stages per pipeline stage: None
|
| 30 |
+
accumulate and all-reduce gradients in fp32 for bfloat16 data type.
|
| 31 |
+
using torch.bfloat16 for parameters ...
|
| 32 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 33 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 34 |
+
------------------------ arguments ------------------------
|
| 35 |
+
account_for_embedding_in_pipeline_split ......... False
|
| 36 |
+
account_for_loss_in_pipeline_split .............. False
|
| 37 |
+
accumulate_allreduce_grads_in_fp32 .............. True
|
| 38 |
+
adam_beta1 ...................................... 0.9
|
| 39 |
+
adam_beta2 ...................................... 0.999
|
| 40 |
+
adam_eps ........................................ 1e-08
|
| 41 |
+
add_bias_linear ................................. False
|
| 42 |
+
add_position_embedding .......................... False
|
| 43 |
+
add_qkv_bias .................................... False
|
| 44 |
+
adlr_autoresume ................................. False
|
| 45 |
+
adlr_autoresume_interval ........................ 1000
|
| 46 |
+
align_grad_reduce ............................... True
|
| 47 |
+
align_param_gather .............................. False
|
| 48 |
+
app_tag_run_name ................................ None
|
| 49 |
+
app_tag_run_version ............................. 0.0.0
|
| 50 |
+
apply_layernorm_1p .............................. False
|
| 51 |
+
apply_query_key_layer_scaling ................... False
|
| 52 |
+
apply_residual_connection_post_layernorm ........ False
|
| 53 |
+
apply_rope_fusion ............................... False
|
| 54 |
+
async_save ...................................... None
|
| 55 |
+
async_tensor_model_parallel_allreduce ........... True
|
| 56 |
+
attention_backend ............................... AttnBackend.auto
|
| 57 |
+
attention_dropout ............................... 0.0
|
| 58 |
+
attention_softmax_in_fp32 ....................... False
|
| 59 |
+
auto_detect_ckpt_format ......................... False
|
| 60 |
+
barrier_with_L1_time ............................ True
|
| 61 |
+
bert_binary_head ................................ True
|
| 62 |
+
bert_embedder_type .............................. megatron
|
| 63 |
+
bert_load ....................................... None
|
| 64 |
+
bf16 ............................................ True
|
| 65 |
+
bias_dropout_fusion ............................. True
|
| 66 |
+
bias_gelu_fusion ................................ False
|
| 67 |
+
bias_swiglu_fusion .............................. True
|
| 68 |
+
biencoder_projection_dim ........................ 0
|
| 69 |
+
biencoder_shared_query_context_model ............ False
|
| 70 |
+
block_data_path ................................. None
|
| 71 |
+
calc_ft_timeouts ................................ False
|
| 72 |
+
calculate_per_token_loss ........................ False
|
| 73 |
+
check_for_large_grads ........................... False
|
| 74 |
+
check_for_nan_in_loss_and_grad .................. False
|
| 75 |
+
check_for_spiky_loss ............................ False
|
| 76 |
+
check_weight_hash_across_dp_replicas_interval ... None
|
| 77 |
+
ckpt_assume_constant_structure .................. False
|
| 78 |
+
ckpt_convert_format ............................. None
|
| 79 |
+
ckpt_convert_save ............................... None
|
| 80 |
+
ckpt_convert_update_legacy_dist_opt_format ...... False
|
| 81 |
+
ckpt_format ..................................... torch_dist
|
| 82 |
+
ckpt_fully_parallel_load ........................ False
|
| 83 |
+
ckpt_fully_parallel_save ........................ True
|
| 84 |
+
ckpt_fully_parallel_save_deprecated ............. False
|
| 85 |
+
ckpt_step ....................................... None
|
| 86 |
+
classes_fraction ................................ 1.0
|
| 87 |
+
clip_grad ....................................... 1.0
|
| 88 |
+
clone_scatter_output_in_embedding ............... True
|
| 89 |
+
config_logger_dir ...............................
|
| 90 |
+
consumed_train_samples .......................... 0
|
| 91 |
+
consumed_valid_samples .......................... 0
|
| 92 |
+
context_parallel_size ........................... 1
|
| 93 |
+
cp_comm_type .................................... ['p2p']
|
| 94 |
+
create_attention_mask_in_dataloader ............. True
|
| 95 |
+
cross_entropy_fusion_impl ....................... native
|
| 96 |
+
cross_entropy_loss_fusion ....................... False
|
| 97 |
+
cuda_graph_scope ................................ full_iteration
|
| 98 |
+
cuda_graph_warmup_steps ......................... 3
|
| 99 |
+
data_args_path .................................. None
|
| 100 |
+
data_cache_path ................................. None
|
| 101 |
+
data_parallel_random_init ....................... False
|
| 102 |
+
data_parallel_sharding_strategy ................. no_shard
|
| 103 |
+
data_parallel_size .............................. 8
|
| 104 |
+
data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document']
|
| 105 |
+
data_per_class_fraction ......................... 1.0
|
| 106 |
+
data_sharding ................................... True
|
| 107 |
+
dataloader_type ................................. single
|
| 108 |
+
ddp_average_in_collective ....................... False
|
| 109 |
+
ddp_bucket_size ................................. None
|
| 110 |
+
ddp_num_buckets ................................. None
|
| 111 |
+
ddp_pad_buckets_for_high_nccl_busbw ............. False
|
| 112 |
+
decoder_first_pipeline_num_layers ............... None
|
| 113 |
+
decoder_last_pipeline_num_layers ................ None
|
| 114 |
+
decoder_num_layers .............................. None
|
| 115 |
+
decoder_seq_length .............................. None
|
| 116 |
+
decoupled_lr .................................... None
|
| 117 |
+
decoupled_min_lr ................................ None
|
| 118 |
+
decrease_batch_size_if_needed ................... False
|
| 119 |
+
defer_embedding_wgrad_compute ................... False
|
| 120 |
+
delay_wgrad_compute ............................. False
|
| 121 |
+
deprecated_use_mcore_models ..................... True
|
| 122 |
+
deterministic_mode .............................. False
|
| 123 |
+
dino_bottleneck_size ............................ 256
|
| 124 |
+
dino_freeze_last_layer .......................... 1
|
| 125 |
+
dino_head_hidden_size ........................... 2048
|
| 126 |
+
dino_local_crops_number ......................... 10
|
| 127 |
+
dino_local_img_size ............................. 96
|
| 128 |
+
dino_norm_last_layer ............................ False
|
| 129 |
+
dino_teacher_temp ............................... 0.07
|
| 130 |
+
dino_warmup_teacher_temp ........................ 0.04
|
| 131 |
+
dino_warmup_teacher_temp_epochs ................. 30
|
| 132 |
+
disable_bf16_reduced_precision_matmul ........... False
|
| 133 |
+
disable_mamba_mem_eff_path ...................... False
|
| 134 |
+
disable_straggler_on_startup .................... False
|
| 135 |
+
dist_ckpt_format_deprecated ..................... None
|
| 136 |
+
dist_ckpt_strictness ............................ assume_ok_unexpected
|
| 137 |
+
distribute_saved_activations .................... False
|
| 138 |
+
distributed_backend ............................. nccl
|
| 139 |
+
distributed_timeout_minutes ..................... 10
|
| 140 |
+
embedding_init_method_std ....................... None
|
| 141 |
+
embedding_path .................................. None
|
| 142 |
+
empty_unused_memory_level ....................... 0
|
| 143 |
+
enable_cuda_graph ............................... True
|
| 144 |
+
enable_experimental ............................. False
|
| 145 |
+
enable_ft_package ............................... False
|
| 146 |
+
enable_gloo_process_groups ...................... True
|
| 147 |
+
enable_msc ...................................... True
|
| 148 |
+
enable_one_logger ............................... True
|
| 149 |
+
encoder_num_layers .............................. 2
|
| 150 |
+
encoder_seq_length .............................. 4096
|
| 151 |
+
end_weight_decay ................................ 0.1
|
| 152 |
+
eod_mask_loss ................................... False
|
| 153 |
+
error_injection_rate ............................ 0
|
| 154 |
+
error_injection_type ............................ transient_error
|
| 155 |
+
eval_interval ................................... 1000000
|
| 156 |
+
eval_iters ...................................... 0
|
| 157 |
+
evidence_data_path .............................. None
|
| 158 |
+
exit_duration_in_mins ........................... None
|
| 159 |
+
exit_interval ................................... None
|
| 160 |
+
exit_on_missing_checkpoint ...................... False
|
| 161 |
+
exit_signal_handler ............................. False
|
| 162 |
+
exp_avg_dtype ................................... torch.float32
|
| 163 |
+
exp_avg_sq_dtype ................................ torch.float32
|
| 164 |
+
expert_model_parallel_size ...................... 8
|
| 165 |
+
expert_tensor_parallel_size ..................... 1
|
| 166 |
+
export_force_local_attention .................... False
|
| 167 |
+
export_kd_cfg ................................... None
|
| 168 |
+
export_kd_teacher_ckpt_format ................... None
|
| 169 |
+
export_kd_teacher_load .......................... None
|
| 170 |
+
export_kv_cache_quant ........................... False
|
| 171 |
+
export_legacy_megatron .......................... False
|
| 172 |
+
export_model_type ............................... GPTModel
|
| 173 |
+
export_moe_apply_probs_on_input ................. False
|
| 174 |
+
export_qk_l2_norm ............................... False
|
| 175 |
+
export_quant_cfg ................................ None
|
| 176 |
+
export_real_quant_cfg ........................... None
|
| 177 |
+
export_te_mcore_model ........................... False
|
| 178 |
+
external_cuda_graph ............................. False
|
| 179 |
+
ffn_hidden_size ................................. 18432
|
| 180 |
+
finetune ........................................ False
|
| 181 |
+
finetune_data_split ............................. train
|
| 182 |
+
finetune_hf_dataset ............................. None
|
| 183 |
+
first_last_layers_bf16 .......................... False
|
| 184 |
+
flash_decode .................................... False
|
| 185 |
+
fp16 ............................................ False
|
| 186 |
+
fp16_lm_cross_entropy ........................... False
|
| 187 |
+
fp32_residual_connection ........................ False
|
| 188 |
+
fp8 ............................................. e4m3
|
| 189 |
+
fp8_amax_compute_algo ........................... most_recent
|
| 190 |
+
fp8_amax_history_len ............................ 1
|
| 191 |
+
fp8_interval .................................... 1
|
| 192 |
+
fp8_margin ...................................... 0
|
| 193 |
+
fp8_param_gather ................................ False
|
| 194 |
+
fp8_recipe ...................................... mxfp8
|
| 195 |
+
fp8_wgrad ....................................... True
|
| 196 |
+
fsdp_double_buffer .............................. False
|
| 197 |
+
global_batch_size ............................... 8
|
| 198 |
+
grad_reduce_in_bf16 ............................. False
|
| 199 |
+
gradient_accumulation_fusion .................... True
|
| 200 |
+
gradient_reduce_div_fusion ...................... True
|
| 201 |
+
group_query_attention ........................... False
|
| 202 |
+
head_lr_mult .................................... 1.0
|
| 203 |
+
heterogeneous_layers_config_encoded_json ........ None
|
| 204 |
+
heterogeneous_layers_config_path ................ None
|
| 205 |
+
hidden_dropout .................................. 0.0
|
| 206 |
+
hidden_size ..................................... 7168
|
| 207 |
+
hierarchical_context_parallel_sizes ............. None
|
| 208 |
+
high_priority_stream_groups ..................... []
|
| 209 |
+
hybrid_attention_ratio .......................... 0.0
|
| 210 |
+
hybrid_mlp_ratio ................................ 0.0
|
| 211 |
+
hybrid_override_pattern ......................... None
|
| 212 |
+
hysteresis ...................................... 2
|
| 213 |
+
ict_head_size ................................... None
|
| 214 |
+
ict_load ........................................ None
|
| 215 |
+
img_h ........................................... 224
|
| 216 |
+
img_w ........................................... 224
|
| 217 |
+
indexer_batch_size .............................. 128
|
| 218 |
+
indexer_log_interval ............................ 1000
|
| 219 |
+
inference_batch_times_seqlen_threshold .......... -1
|
| 220 |
+
inference_dynamic_batching ...................... False
|
| 221 |
+
inference_dynamic_batching_buffer_guaranteed_fraction 0.2
|
| 222 |
+
inference_dynamic_batching_buffer_overflow_factor None
|
| 223 |
+
inference_dynamic_batching_buffer_size_gb ....... 40.0
|
| 224 |
+
inference_dynamic_batching_chunk_size ........... 256
|
| 225 |
+
inference_dynamic_batching_max_requests_override None
|
| 226 |
+
inference_dynamic_batching_max_tokens_override .. None
|
| 227 |
+
inference_dynamic_batching_num_cuda_graphs ...... 16
|
| 228 |
+
inference_max_batch_size ........................ 8
|
| 229 |
+
inference_max_seq_length ........................ 2560
|
| 230 |
+
inference_rng_tracker ........................... False
|
| 231 |
+
init_method_std ................................. 0.02
|
| 232 |
+
init_method_xavier_uniform ...................... False
|
| 233 |
+
init_model_with_meta_device ..................... False
|
| 234 |
+
initial_loss_scale .............................. 4294967296
|
| 235 |
+
inprocess_active_world_size ..................... 8
|
| 236 |
+
inprocess_barrier_timeout ....................... 120
|
| 237 |
+
inprocess_completion_timeout .................... 120
|
| 238 |
+
inprocess_empty_cuda_cache ...................... False
|
| 239 |
+
inprocess_granularity ........................... node
|
| 240 |
+
inprocess_hard_timeout .......................... 90
|
| 241 |
+
inprocess_heartbeat_interval .................... 30
|
| 242 |
+
inprocess_heartbeat_timeout ..................... 60
|
| 243 |
+
inprocess_last_call_wait ........................ 1
|
| 244 |
+
inprocess_max_iterations ........................ None
|
| 245 |
+
inprocess_monitor_process_interval .............. 1.0
|
| 246 |
+
inprocess_monitor_thread_interval ............... 1.0
|
| 247 |
+
inprocess_progress_watchdog_interval ............ 1.0
|
| 248 |
+
inprocess_restart ............................... False
|
| 249 |
+
inprocess_soft_timeout .......................... 60
|
| 250 |
+
inprocess_termination_grace_time ................ 1
|
| 251 |
+
is_hybrid_model ................................. False
|
| 252 |
+
iter_per_epoch .................................. 1250
|
| 253 |
+
iterations_to_skip .............................. []
|
| 254 |
+
keep_fp8_transpose_cache_when_using_custom_fsdp . False
|
| 255 |
+
kitchen_config_file ............................. None
|
| 256 |
+
kitchen_recipe_number ........................... None
|
| 257 |
+
kv_channels ..................................... 64
|
| 258 |
+
kv_lora_rank .................................... 512
|
| 259 |
+
lazy_mpu_init ................................... None
|
| 260 |
+
load ............................................ None
|
| 261 |
+
load_main_params_from_ckpt ...................... None
|
| 262 |
+
load_model_opt_format ........................... False
|
| 263 |
+
local_rank ...................................... 0
|
| 264 |
+
log_energy ...................................... False
|
| 265 |
+
log_interval .................................... 1
|
| 266 |
+
log_loss_scale_to_tensorboard ................... True
|
| 267 |
+
log_memory_to_tensorboard ....................... False
|
| 268 |
+
log_num_zeros_in_grad ........................... False
|
| 269 |
+
log_params_norm ................................. False
|
| 270 |
+
log_progress .................................... False
|
| 271 |
+
log_straggler ................................... False
|
| 272 |
+
log_throughput .................................. True
|
| 273 |
+
log_timers_to_tensorboard ....................... False
|
| 274 |
+
log_validation_ppl_to_tensorboard ............... False
|
| 275 |
+
log_world_size_to_tensorboard ................... False
|
| 276 |
+
logging_level ................................... None
|
| 277 |
+
loss_scale ...................................... None
|
| 278 |
+
loss_scale_window ............................... 1000
|
| 279 |
+
lr .............................................. 0.0001
|
| 280 |
+
lr_decay_iters .................................. 8
|
| 281 |
+
lr_decay_samples ................................ None
|
| 282 |
+
lr_decay_style .................................. cosine
|
| 283 |
+
lr_warmup_fraction .............................. None
|
| 284 |
+
lr_warmup_init .................................. 0.0
|
| 285 |
+
lr_warmup_iters ................................. 5
|
| 286 |
+
lr_warmup_samples ............................... 0
|
| 287 |
+
lr_wsd_decay_iters .............................. None
|
| 288 |
+
lr_wsd_decay_samples ............................ None
|
| 289 |
+
lr_wsd_decay_style .............................. exponential
|
| 290 |
+
main_grads_dtype ................................ torch.float32
|
| 291 |
+
main_params_dtype ............................... torch.float32
|
| 292 |
+
make_vocab_size_divisible_by .................... 128
|
| 293 |
+
mamba_head_dim .................................. 64
|
| 294 |
+
mamba_num_groups ................................ 8
|
| 295 |
+
mamba_num_heads ................................. None
|
| 296 |
+
mamba_state_dim ................................. 128
|
| 297 |
+
manual_gc ....................................... False
|
| 298 |
+
manual_gc_eval .................................. True
|
| 299 |
+
manual_gc_interval .............................. 0
|
| 300 |
+
mask_factor ..................................... 1.0
|
| 301 |
+
mask_prob ....................................... 0.15
|
| 302 |
+
mask_type ....................................... random
|
| 303 |
+
masked_softmax_fusion ........................... False
|
| 304 |
+
max_position_embeddings ......................... 4096
|
| 305 |
+
max_tokens_to_oom ............................... 12000
|
| 306 |
+
memory_snapshot_path ............................ snapshot.pickle
|
| 307 |
+
merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt
|
| 308 |
+
micro_batch_size ................................ 1
|
| 309 |
+
microbatch_group_size_per_vp_stage .............. None
|
| 310 |
+
mid_level_dataset_surplus ....................... 0.005
|
| 311 |
+
min_loss_scale .................................. 1.0
|
| 312 |
+
min_lr .......................................... 1e-05
|
| 313 |
+
mlp_chunks_for_prefill .......................... 1
|
| 314 |
+
mmap_bin_files .................................. True
|
| 315 |
+
mock_data ....................................... False
|
| 316 |
+
moe_apply_probs_on_input ........................ False
|
| 317 |
+
moe_aux_loss_coeff .............................. 0.0
|
| 318 |
+
moe_capacity_monitor ............................ True
|
| 319 |
+
moe_deepep_num_sms .............................. 20
|
| 320 |
+
moe_echo_enable_random_offloading ............... False
|
| 321 |
+
moe_echo_expert_dispatch_overlap ................ False
|
| 322 |
+
moe_echo_expert_dispatcher_type ................. hybridep
|
| 323 |
+
moe_echo_recompute_expert_dispatch .............. False
|
| 324 |
+
moe_enable_deepep ............................... False
|
| 325 |
+
moe_enable_echo ................................. False
|
| 326 |
+
moe_enable_hybridep ............................. True
|
| 327 |
+
moe_enable_ultraep .............................. True
|
| 328 |
+
moe_expert_capacity_factor ...................... None
|
| 329 |
+
moe_extended_tp ................................. False
|
| 330 |
+
moe_ffn_hidden_size ............................. 2048
|
| 331 |
+
moe_grouped_gemm ................................ True
|
| 332 |
+
moe_hybridep_num_sms ............................ 16
|
| 333 |
+
moe_input_jitter_eps ............................ None
|
| 334 |
+
moe_layer_freq .................................. [0, 1]
|
| 335 |
+
moe_layer_recompute ............................. False
|
| 336 |
+
moe_num_echo_experts ............................ None
|
| 337 |
+
moe_num_redundant_experts_per_rank .............. 48
|
| 338 |
+
moe_pad_expert_input_to_capacity ................ False
|
| 339 |
+
moe_per_layer_logging ........................... False
|
| 340 |
+
moe_permute_fusion .............................. False
|
| 341 |
+
moe_received_token_capacity ..................... 1.17578125
|
| 342 |
+
moe_router_bias_update_rate ..................... 0.001
|
| 343 |
+
moe_router_dtype ................................ fp32
|
| 344 |
+
moe_router_enable_expert_bias ................... True
|
| 345 |
+
moe_router_force_load_balancing ................. False
|
| 346 |
+
moe_router_group_topk ........................... 1
|
| 347 |
+
moe_router_load_balancing_type .................. seq_aux_loss
|
| 348 |
+
moe_router_num_groups ........................... 1
|
| 349 |
+
moe_router_padding_for_fp8 ...................... False
|
| 350 |
+
moe_router_pre_softmax .......................... False
|
| 351 |
+
moe_router_score_function ....................... sigmoid
|
| 352 |
+
moe_router_topk ................................. 8
|
| 353 |
+
moe_router_topk_scaling_factor .................. 2.827
|
| 354 |
+
moe_shared_expert_intermediate_size ............. 2048
|
| 355 |
+
moe_shared_expert_overlap ....................... False
|
| 356 |
+
moe_token_dispatcher_type ....................... flex
|
| 357 |
+
moe_token_drop_policy ........................... probs
|
| 358 |
+
moe_ultraep_max_microbatches .................... None
|
| 359 |
+
moe_ultraep_overlap_weight_sync ................. True
|
| 360 |
+
moe_ultraep_weight_sync_join .................... before_dispatch
|
| 361 |
+
moe_upcycling_granularity ....................... 1
|
| 362 |
+
moe_use_device_initiated_grouped_gemm ........... True
|
| 363 |
+
moe_use_legacy_grouped_gemm ..................... False
|
| 364 |
+
moe_use_upcycling ............................... False
|
| 365 |
+
moe_z_loss_coeff ................................ None
|
| 366 |
+
mrope_section ................................... None
|
| 367 |
+
mscale .......................................... 1.0
|
| 368 |
+
mscale_all_dim .................................. 1.0
|
| 369 |
+
mtp_loss_scaling_factor ......................... 0.1
|
| 370 |
+
mtp_num_layers .................................. None
|
| 371 |
+
multi_latent_attention .......................... True
|
| 372 |
+
nccl_all_reduce_for_prefill ..................... False
|
| 373 |
+
nccl_communicator_config_path ................... None
|
| 374 |
+
nccl_ub ......................................... False
|
| 375 |
+
no_load_optim ................................... None
|
| 376 |
+
no_load_rng ..................................... None
|
| 377 |
+
no_persist_layer_norm ........................... False
|
| 378 |
+
no_rope_freq .................................... None
|
| 379 |
+
no_save_optim ................................... None
|
| 380 |
+
no_save_rng ..................................... None
|
| 381 |
+
non_persistent_ckpt_type ........................ None
|
| 382 |
+
non_persistent_global_ckpt_dir .................. None
|
| 383 |
+
non_persistent_local_ckpt_algo .................. fully_parallel
|
| 384 |
+
non_persistent_local_ckpt_dir ................... None
|
| 385 |
+
non_persistent_save_interval .................... None
|
| 386 |
+
norm_epsilon .................................... 1e-06
|
| 387 |
+
normalization ................................... RMSNorm
|
| 388 |
+
num_attention_heads ............................. 64
|
| 389 |
+
num_channels .................................... 3
|
| 390 |
+
num_classes ..................................... 1000
|
| 391 |
+
num_dataset_builder_threads ..................... 1
|
| 392 |
+
num_distributed_optimizer_instances ............. 1
|
| 393 |
+
num_experts ..................................... 384
|
| 394 |
+
num_layers ...................................... 2
|
| 395 |
+
num_layers_at_end_in_bf16 ....................... 1
|
| 396 |
+
num_layers_at_start_in_bf16 ..................... 1
|
| 397 |
+
num_layers_per_virtual_pipeline_stage ........... None
|
| 398 |
+
num_query_groups ................................ 1
|
| 399 |
+
num_virtual_stages_per_pipeline_rank ............ None
|
| 400 |
+
num_workers ..................................... 2
|
| 401 |
+
object_storage_cache_path ....................... None
|
| 402 |
+
one_logger_async ................................ False
|
| 403 |
+
one_logger_project .............................. megatron-lm
|
| 404 |
+
one_logger_run_name ............................. None
|
| 405 |
+
onnx_safe ....................................... None
|
| 406 |
+
openai_gelu ..................................... False
|
| 407 |
+
optimizer ....................................... adam
|
| 408 |
+
optimizer_cpu_offload ........................... False
|
| 409 |
+
optimizer_offload_fraction ...................... 1.0
|
| 410 |
+
output_bert_embeddings .......................... False
|
| 411 |
+
overlap_cpu_optimizer_d2h_h2d ................... False
|
| 412 |
+
overlap_grad_reduce ............................. True
|
| 413 |
+
overlap_moe_expert_parallel_comm ................ False
|
| 414 |
+
overlap_p2p_comm ................................ False
|
| 415 |
+
overlap_p2p_comm_warmup_flush ................... False
|
| 416 |
+
overlap_param_gather ............................ True
|
| 417 |
+
overlap_param_gather_with_optimizer_step ........ False
|
| 418 |
+
override_opt_param_scheduler .................... False
|
| 419 |
+
params_dtype .................................... torch.bfloat16
|
| 420 |
+
patch_dim ....................................... 16
|
| 421 |
+
per_split_data_args_path ........................ None
|
| 422 |
+
perform_initialization .......................... True
|
| 423 |
+
pin_cpu_grads ................................... True
|
| 424 |
+
pin_cpu_params .................................. True
|
| 425 |
+
pipeline_model_parallel_comm_backend ............ None
|
| 426 |
+
pipeline_model_parallel_layout .................. None
|
| 427 |
+
pipeline_model_parallel_size .................... 1
|
| 428 |
+
position_embedding_type ......................... rope
|
| 429 |
+
pretrained_checkpoint ........................... None
|
| 430 |
+
profile ......................................... False
|
| 431 |
+
profile_ranks ................................... [0]
|
| 432 |
+
profile_step_end ................................ 12
|
| 433 |
+
profile_step_start .............................. 10
|
| 434 |
+
q_lora_rank ..................................... 1536
|
| 435 |
+
qk_head_dim ..................................... 128
|
| 436 |
+
qk_l2_norm ...................................... False
|
| 437 |
+
qk_layernorm .................................... True
|
| 438 |
+
qk_pos_emb_head_dim ............................. 64
|
| 439 |
+
query_in_block_prob ............................. 0.1
|
| 440 |
+
rampup_batch_size ............................... None
|
| 441 |
+
rank ............................................ 0
|
| 442 |
+
recompute_granularity ........................... None
|
| 443 |
+
recompute_method ................................ None
|
| 444 |
+
recompute_modules ............................... None
|
| 445 |
+
recompute_num_layers ............................ None
|
| 446 |
+
record_memory_history ........................... False
|
| 447 |
+
relative_attention_max_distance ................. 128
|
| 448 |
+
relative_attention_num_buckets .................. 32
|
| 449 |
+
replication ..................................... False
|
| 450 |
+
replication_factor .............................. 2
|
| 451 |
+
replication_jump ................................ None
|
| 452 |
+
rerun_mode ...................................... validate_results
|
| 453 |
+
reset_attention_mask ............................ False
|
| 454 |
+
reset_position_ids .............................. False
|
| 455 |
+
result_rejected_tracker_filename ................ None
|
| 456 |
+
retriever_report_topk_accuracies ................ []
|
| 457 |
+
retriever_score_scaling ......................... False
|
| 458 |
+
retriever_seq_length ............................ 256
|
| 459 |
+
retro_add_retriever ............................. False
|
| 460 |
+
retro_attention_gate ............................ 1
|
| 461 |
+
retro_cyclic_train_iters ........................ None
|
| 462 |
+
retro_encoder_attention_dropout ................. 0.1
|
| 463 |
+
retro_encoder_hidden_dropout .................... 0.1
|
| 464 |
+
retro_encoder_layers ............................ 2
|
| 465 |
+
retro_num_neighbors ............................. 2
|
| 466 |
+
retro_num_retrieved_chunks ...................... 2
|
| 467 |
+
retro_project_dir ............................... None
|
| 468 |
+
retro_verify_neighbor_count ..................... True
|
| 469 |
+
reuse_grad_buf_for_mxfp8_param_ag ............... False
|
| 470 |
+
rope_scaling_factor ............................. 8.0
|
| 471 |
+
rope_type ....................................... None
|
| 472 |
+
rotary_base ..................................... 50000
|
| 473 |
+
rotary_interleaved .............................. False
|
| 474 |
+
rotary_percent .................................. 1.0
|
| 475 |
+
rotary_scaling_factor ........................... 32.0
|
| 476 |
+
rotary_seq_len_interpolation_factor ............. None
|
| 477 |
+
run_workload_inspector_server ................... False
|
| 478 |
+
sample_rate ..................................... 1.0
|
| 479 |
+
save ............................................ None
|
| 480 |
+
save_interval ................................... None
|
| 481 |
+
save_retain_interval ............................ None
|
| 482 |
+
scatter_gather_tensors_in_pipeline .............. True
|
| 483 |
+
seed ............................................ 1234
|
| 484 |
+
seq_length ...................................... 4096
|
| 485 |
+
sequence_parallel ............................... False
|
| 486 |
+
sft ............................................. False
|
| 487 |
+
sft_tokenizer_prompt_format ..................... nemotron-h-aligned
|
| 488 |
+
sgd_momentum .................................... 0.9
|
| 489 |
+
short_seq_prob .................................. 0.1
|
| 490 |
+
skip_train ...................................... False
|
| 491 |
+
skipped_train_samples ........................... 0
|
| 492 |
+
spec ............................................ None
|
| 493 |
+
split ........................................... 100,0,0
|
| 494 |
+
squared_relu .................................... False
|
| 495 |
+
start_weight_decay .............................. 0.1
|
| 496 |
+
straggler_ctrlr_port ............................ 65535
|
| 497 |
+
straggler_minmax_count .......................... 1
|
| 498 |
+
suggested_communication_unit_size ............... None
|
| 499 |
+
swiglu .......................................... True
|
| 500 |
+
swin_backbone_type .............................. tiny
|
| 501 |
+
symmetric_ar_type ............................... None
|
| 502 |
+
te_rng_tracker .................................. True
|
| 503 |
+
tensor_model_parallel_size ...................... 1
|
| 504 |
+
tensorboard_dir ................................. None
|
| 505 |
+
tensorboard_log_interval ........................ 1
|
| 506 |
+
tensorboard_queue_size .......................... 1000
|
| 507 |
+
test_data_path .................................. None
|
| 508 |
+
test_mode ....................................... False
|
| 509 |
+
tiktoken_num_special_tokens ..................... 1000
|
| 510 |
+
tiktoken_pattern ................................ None
|
| 511 |
+
tiktoken_special_tokens ......................... None
|
| 512 |
+
timing_log_level ................................ 0
|
| 513 |
+
timing_log_option ............................... minmax
|
| 514 |
+
titles_data_path ................................ None
|
| 515 |
+
tokenizer_model ................................. None
|
| 516 |
+
tokenizer_type .................................. GPT2BPETokenizer
|
| 517 |
+
torch_fsdp2_reshard_after_forward ............... True
|
| 518 |
+
tp_comm_bootstrap_backend ....................... nccl
|
| 519 |
+
tp_comm_bulk_dgrad .............................. True
|
| 520 |
+
tp_comm_bulk_wgrad .............................. True
|
| 521 |
+
tp_comm_overlap ................................. False
|
| 522 |
+
tp_comm_overlap_ag .............................. True
|
| 523 |
+
tp_comm_overlap_cfg ............................. None
|
| 524 |
+
tp_comm_overlap_rs .............................. True
|
| 525 |
+
tp_comm_overlap_rs_dgrad ........................ False
|
| 526 |
+
tp_comm_split_ag ................................ True
|
| 527 |
+
tp_comm_split_rs ................................ True
|
| 528 |
+
train_data_path ................................. None
|
| 529 |
+
train_iters ..................................... 12
|
| 530 |
+
train_samples ................................... None
|
| 531 |
+
train_sync_interval ............................. None
|
| 532 |
+
transformer_impl ................................ transformer_engine
|
| 533 |
+
transformer_pipeline_model_parallel_size ........ 1
|
| 534 |
+
untie_embeddings_and_output_weights ............. True
|
| 535 |
+
use_checkpoint_args ............................. False
|
| 536 |
+
use_checkpoint_opt_param_scheduler .............. False
|
| 537 |
+
use_cpu_initialization .......................... None
|
| 538 |
+
use_custom_fsdp ................................. False
|
| 539 |
+
use_dist_ckpt ................................... True
|
| 540 |
+
use_dist_ckpt_deprecated ........................ False
|
| 541 |
+
use_distributed_optimizer ....................... True
|
| 542 |
+
use_flash_attn .................................. False
|
| 543 |
+
use_legacy_models ............................... False
|
| 544 |
+
use_mp_args_from_checkpoint_args ................ False
|
| 545 |
+
use_one_sent_docs ............................... False
|
| 546 |
+
use_persistent_ckpt_worker ...................... False
|
| 547 |
+
use_precision_aware_optimizer ................... False
|
| 548 |
+
use_pytorch_profiler ............................ False
|
| 549 |
+
use_ring_exchange_p2p ........................... False
|
| 550 |
+
use_rope_scaling ................................ False
|
| 551 |
+
use_rotary_position_embeddings .................. False
|
| 552 |
+
use_sharp ....................................... False
|
| 553 |
+
use_tokenizer_model_from_checkpoint_args ........ True
|
| 554 |
+
use_torch_fsdp2 ................................. False
|
| 555 |
+
use_torch_optimizer_for_cpu_offload ............. False
|
| 556 |
+
use_tp_pp_dp_mapping ............................ False
|
| 557 |
+
v_head_dim ...................................... 128
|
| 558 |
+
valid_data_path ................................. None
|
| 559 |
+
variable_seq_lengths ............................ False
|
| 560 |
+
virtual_pipeline_model_parallel_size ............ None
|
| 561 |
+
vision_backbone_type ............................ vit
|
| 562 |
+
vision_pretraining .............................. False
|
| 563 |
+
vision_pretraining_type ......................... classify
|
| 564 |
+
vocab_extra_ids ................................. 0
|
| 565 |
+
vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json
|
| 566 |
+
vocab_size ...................................... None
|
| 567 |
+
wandb_exp_name ..................................
|
| 568 |
+
wandb_project ...................................
|
| 569 |
+
wandb_save_dir ..................................
|
| 570 |
+
weight_decay .................................... 0.1
|
| 571 |
+
weight_decay_incr_style ......................... constant
|
| 572 |
+
wgrad_deferral_limit ............................ 0
|
| 573 |
+
world_size ...................................... 8
|
| 574 |
+
yaml_cfg ........................................ None
|
| 575 |
+
-------------------- end of arguments ---------------------
|
| 576 |
+
INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1
|
| 577 |
+
> building GPT2BPETokenizer tokenizer ...
|
| 578 |
+
> padded vocab (size: 50257) with 47 dummy tokens (new size: 50304)
|
| 579 |
+
WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS
|
| 580 |
+
> initializing torch distributed ...
|
| 581 |
+
/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
|
| 582 |
+
warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
|
| 583 |
+
[Gloo] Rank [Gloo] Rank 2 is connected to 7 peer ranks. 1Expected number of connected peer ranks is : is connected to 77
|
| 584 |
+
peer ranks. Expected number of connected peer ranks is : 7
|
| 585 |
+
[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 586 |
+
[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 587 |
+
[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 588 |
+
[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 589 |
+
[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 590 |
+
[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 591 |
+
[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 592 |
+
[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 593 |
+
[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
|
| 594 |
+
[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 25467 is connected to is connected to is connected to is connected to is connected to 77777 peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77777
|
| 595 |
+
|
| 596 |
+
|
| 597 |
+
|
| 598 |
+
|
| 599 |
+
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
|
| 600 |
+
> initialized tensor model parallel with size 1
|
| 601 |
+
> initialized pipeline model parallel with size 1
|
| 602 |
+
> setting random seeds to 1234 ...
|
| 603 |
+
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
|
| 604 |
+
[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00
|
| 605 |
+
|
| 606 |
+
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
|
| 607 |
+
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
|
| 608 |
+
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
|
| 609 |
+
[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
|
| 610 |
+
[rank6]:[W921 07:49:55.489030863 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 611 |
+
[rank3]:[W921 07:49:55.489174158 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 612 |
+
[rank2]:[W921 07:49:55.489180926 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 613 |
+
[rank1]:[W921 07:49:55.489186964 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 614 |
+
[rank5]:[W921 07:49:55.489214517 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 615 |
+
[rank7]:[W921 07:49:55.489255123 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 616 |
+
[rank4]:[W921 07:49:55.489344079 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 617 |
+
> compiling dataset index builder ...
|
| 618 |
+
make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
|
| 619 |
+
make: Nothing to be done for 'default'.
|
| 620 |
+
make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
|
| 621 |
+
>>> done with dataset index builder. Compilation time: 0.036 seconds
|
| 622 |
+
WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations.
|
| 623 |
+
> compiling and loading fused kernels ...
|
| 624 |
+
[rank0]:[W921 07:49:55.651074971 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
|
| 625 |
+
>>> done with compiling and loading fused kernels. Compilation time: 4.296 seconds
|
| 626 |
+
time to initialize megatron (seconds): 7.944
|
| 627 |
+
[after megatron is initialized] datetime: 2026-09-21 07:50:01
|
| 628 |
+
building GPT model ...
|
| 629 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 630 |
+
warnings.warn(
|
| 631 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 632 |
+
warnings.warn(
|
| 633 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 634 |
+
warnings.warn(
|
| 635 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 636 |
+
warnings.warn(
|
| 637 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 638 |
+
warnings.warn(
|
| 639 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 640 |
+
warnings.warn(
|
| 641 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 642 |
+
warnings.warn(
|
| 643 |
+
/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
|
| 644 |
+
warnings.warn(
|
| 645 |
+
[INFO] Use MNNVL fabric: False
|
| 646 |
+
[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8.
|
| 647 |
+
NVSHMEM v3.7.2
|
| 648 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 649 |
+
|
| 650 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 651 |
+
|
| 652 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 653 |
+
|
| 654 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 655 |
+
|
| 656 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 657 |
+
|
| 658 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 659 |
+
|
| 660 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 661 |
+
|
| 662 |
+
/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed.
|
| 663 |
+
|
| 664 |
+
UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive).
|
| 665 |
+
> number of parameters on (tensor, pipeline) model parallel rank (0, 0): 5594455040
|
| 666 |
+
INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False)
|
| 667 |
+
INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 9
|
| 668 |
+
Params for bucket 1 (360579072 elements, 360579072 padded size):
|
| 669 |
+
module.output_layer.weight
|
| 670 |
+
Params for bucket 2 (44047360 elements, 44047360 padded size):
|
| 671 |
+
module.decoder.final_layernorm.weight
|
| 672 |
+
module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight
|
| 673 |
+
module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight
|
| 674 |
+
Params for bucket 3 (45163520 elements, 45163520 padded size):
|
| 675 |
+
module.decoder.layers.1.mlp.router.weight
|
| 676 |
+
module.decoder.layers.1.pre_mlp_layernorm.weight
|
| 677 |
+
module.decoder.layers.1.self_attention.linear_kv_up_proj.weight
|
| 678 |
+
module.decoder.layers.1.self_attention.linear_q_up_proj.layer_norm_weight
|
| 679 |
+
module.decoder.layers.1.self_attention.linear_q_down_proj.weight
|
| 680 |
+
module.decoder.layers.1.self_attention.linear_kv_down_proj.weight
|
| 681 |
+
module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight
|
| 682 |
+
module.decoder.layers.1.self_attention.linear_q_up_proj.weight
|
| 683 |
+
Params for bucket 4 (58720256 elements, 58720256 padded size):
|
| 684 |
+
module.decoder.layers.1.self_attention.linear_proj.weight
|
| 685 |
+
Params for bucket 5 (132127744 elements, 132127744 padded size):
|
| 686 |
+
module.decoder.layers.1.input_layernorm.weight
|
| 687 |
+
module.decoder.layers.0.mlp.linear_fc2.weight
|
| 688 |
+
Params for bucket 6 (264241152 elements, 264241152 padded size):
|
| 689 |
+
module.decoder.layers.0.mlp.linear_fc1.weight
|
| 690 |
+
Params for bucket 7 (42411008 elements, 42411008 padded size):
|
| 691 |
+
module.decoder.layers.0.self_attention.linear_kv_up_proj.weight
|
| 692 |
+
module.decoder.layers.0.self_attention.linear_kv_down_proj.weight
|
| 693 |
+
module.decoder.layers.0.self_attention.linear_q_down_proj.weight
|
| 694 |
+
module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight
|
| 695 |
+
module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight
|
| 696 |
+
module.decoder.layers.0.self_attention.linear_q_up_proj.weight
|
| 697 |
+
module.decoder.layers.0.self_attention.linear_q_up_proj.layer_norm_weight
|
| 698 |
+
Params for bucket 8 (58720256 elements, 58720256 padded size):
|
| 699 |
+
module.decoder.layers.0.self_attention.linear_proj.weight
|
| 700 |
+
Params for bucket 9 (360586240 elements, 360586240 padded size):
|
| 701 |
+
module.decoder.layers.0.input_layernorm.weight
|
| 702 |
+
module.embedding.word_embeddings.weight
|
| 703 |
+
INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40
|
| 704 |
+
Params for bucket 1 (44040192 elements, 44040192 padded size):
|
| 705 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight47
|
| 706 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight45
|
| 707 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight46
|
| 708 |
+
Params for bucket 2 (44040192 elements, 44040192 padded size):
|
| 709 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight44
|
| 710 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight42
|
| 711 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight43
|
| 712 |
+
Params for bucket 3 (44040192 elements, 44040192 padded size):
|
| 713 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight41
|
| 714 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight39
|
| 715 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight40
|
| 716 |
+
Params for bucket 4 (44040192 elements, 44040192 padded size):
|
| 717 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight38
|
| 718 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight37
|
| 719 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight36
|
| 720 |
+
Params for bucket 5 (44040192 elements, 44040192 padded size):
|
| 721 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight35
|
| 722 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight34
|
| 723 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight33
|
| 724 |
+
Params for bucket 6 (44040192 elements, 44040192 padded size):
|
| 725 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight32
|
| 726 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight31
|
| 727 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight30
|
| 728 |
+
Params for bucket 7 (44040192 elements, 44040192 padded size):
|
| 729 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight29
|
| 730 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight28
|
| 731 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight27
|
| 732 |
+
Params for bucket 8 (44040192 elements, 44040192 padded size):
|
| 733 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight26
|
| 734 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight25
|
| 735 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight24
|
| 736 |
+
Params for bucket 9 (44040192 elements, 44040192 padded size):
|
| 737 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight23
|
| 738 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight21
|
| 739 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight22
|
| 740 |
+
Params for bucket 10 (44040192 elements, 44040192 padded size):
|
| 741 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight20
|
| 742 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight18
|
| 743 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight19
|
| 744 |
+
Params for bucket 11 (44040192 elements, 44040192 padded size):
|
| 745 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight17
|
| 746 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight15
|
| 747 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight16
|
| 748 |
+
Params for bucket 12 (44040192 elements, 44040192 padded size):
|
| 749 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight14
|
| 750 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight12
|
| 751 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight13
|
| 752 |
+
Params for bucket 13 (44040192 elements, 44040192 padded size):
|
| 753 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight11
|
| 754 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight9
|
| 755 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight10
|
| 756 |
+
Params for bucket 14 (44040192 elements, 44040192 padded size):
|
| 757 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight8
|
| 758 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight6
|
| 759 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight7
|
| 760 |
+
Params for bucket 15 (44040192 elements, 44040192 padded size):
|
| 761 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight5
|
| 762 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight3
|
| 763 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight4
|
| 764 |
+
Params for bucket 16 (44040192 elements, 44040192 padded size):
|
| 765 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight2
|
| 766 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight0
|
| 767 |
+
module.decoder.layers.1.mlp.experts.linear_fc2.weight1
|
| 768 |
+
Params for bucket 17 (58720256 elements, 58720256 padded size):
|
| 769 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight47
|
| 770 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight46
|
| 771 |
+
Params for bucket 18 (58720256 elements, 58720256 padded size):
|
| 772 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight45
|
| 773 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight44
|
| 774 |
+
Params for bucket 19 (58720256 elements, 58720256 padded size):
|
| 775 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight43
|
| 776 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight42
|
| 777 |
+
Params for bucket 20 (58720256 elements, 58720256 padded size):
|
| 778 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight41
|
| 779 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight40
|
| 780 |
+
Params for bucket 21 (58720256 elements, 58720256 padded size):
|
| 781 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight39
|
| 782 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight38
|
| 783 |
+
Params for bucket 22 (58720256 elements, 58720256 padded size):
|
| 784 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight37
|
| 785 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight36
|
| 786 |
+
Params for bucket 23 (58720256 elements, 58720256 padded size):
|
| 787 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight35
|
| 788 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight34
|
| 789 |
+
Params for bucket 24 (58720256 elements, 58720256 padded size):
|
| 790 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight33
|
| 791 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight32
|
| 792 |
+
Params for bucket 25 (58720256 elements, 58720256 padded size):
|
| 793 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight31
|
| 794 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight30
|
| 795 |
+
Params for bucket 26 (58720256 elements, 58720256 padded size):
|
| 796 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight29
|
| 797 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight28
|
| 798 |
+
Params for bucket 27 (58720256 elements, 58720256 padded size):
|
| 799 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight27
|
| 800 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight26
|
| 801 |
+
Params for bucket 28 (58720256 elements, 58720256 padded size):
|
| 802 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight25
|
| 803 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight24
|
| 804 |
+
Params for bucket 29 (58720256 elements, 58720256 padded size):
|
| 805 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight23
|
| 806 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight22
|
| 807 |
+
Params for bucket 30 (58720256 elements, 58720256 padded size):
|
| 808 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight21
|
| 809 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight20
|
| 810 |
+
Params for bucket 31 (58720256 elements, 58720256 padded size):
|
| 811 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight19
|
| 812 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight18
|
| 813 |
+
Params for bucket 32 (58720256 elements, 58720256 padded size):
|
| 814 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight17
|
| 815 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight16
|
| 816 |
+
Params for bucket 33 (58720256 elements, 58720256 padded size):
|
| 817 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight15
|
| 818 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight14
|
| 819 |
+
Params for bucket 34 (58720256 elements, 58720256 padded size):
|
| 820 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight13
|
| 821 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight12
|
| 822 |
+
Params for bucket 35 (58720256 elements, 58720256 padded size):
|
| 823 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight11
|
| 824 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight10
|
| 825 |
+
Params for bucket 36 (58720256 elements, 58720256 padded size):
|
| 826 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight9
|
| 827 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight8
|
| 828 |
+
Params for bucket 37 (58720256 elements, 58720256 padded size):
|
| 829 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight7
|
| 830 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight6
|
| 831 |
+
Params for bucket 38 (58720256 elements, 58720256 padded size):
|
| 832 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight5
|
| 833 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight4
|
| 834 |
+
Params for bucket 39 (58720256 elements, 58720256 padded size):
|
| 835 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight3
|
| 836 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight2
|
| 837 |
+
Params for bucket 40 (58720256 elements, 58720256 padded size):
|
| 838 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight1
|
| 839 |
+
module.decoder.layers.1.mlp.experts.linear_fc1.weight0
|
| 840 |
+
INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=<megatron.core.timers.Timers object at 0x76f5d7cfea20>, config_logger_dir='')
|
| 841 |
+
INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine
|
| 842 |
+
[after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-21 07:50:05
|
| 843 |
+
> building train, validation, and test datasets ...
|
| 844 |
+
> datasets target sizes (minimum size):
|
| 845 |
+
train: 96
|
| 846 |
+
validation: 0
|
| 847 |
+
test: 0
|
| 848 |
+
INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None]
|
| 849 |
+
> building train, validation, and test datasets for GPT ...
|
| 850 |
+
INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=<megatron.training.tokenizer.tokenizer._GPT2BPETokenizer object at 0x76f5d7df6ae0>, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None)
|
| 851 |
+
INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx
|
| 852 |
+
INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths
|
| 853 |
+
INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers
|
| 854 |
+
INFO:megatron.core.datasets.indexed_dataset: Extract the document indices
|
| 855 |
+
INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029
|
| 856 |
+
INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350
|
| 857 |
+
INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices
|
| 858 |
+
INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy
|
| 859 |
+
INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy
|
| 860 |
+
INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy
|
| 861 |
+
INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073
|
| 862 |
+
> finished creating GPT datasets ...
|
| 863 |
+
[after dataloaders are built] datetime: 2026-09-21 07:50:05
|
| 864 |
+
done with setup ...
|
| 865 |
+
(min, max) time across ranks (ms):
|
| 866 |
+
model-and-optimizer-setup ......................: (3319.76, 3323.88)
|
| 867 |
+
train/valid/test-data-iterators-setup ..........: (71.88, 92.46)
|
| 868 |
+
training ...
|
| 869 |
+
Overwriting rerun_state_machine.current_iteration from -1 to 0...
|
| 870 |
+
[before the start of training step] datetime: 2026-09-21 07:50:05
|
| 871 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 872 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 873 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 874 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 875 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 876 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 877 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 878 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 879 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 880 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 881 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 882 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 883 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 884 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 885 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 886 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 887 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 888 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 889 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 890 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 891 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 892 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 893 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 894 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 895 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 896 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 897 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 898 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 899 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 900 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 901 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 902 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 903 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 904 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 905 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 906 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 907 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 908 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 909 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 910 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 911 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 912 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 913 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 914 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 915 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 916 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 917 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 918 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 919 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 920 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 921 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 922 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 923 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 924 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 925 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 926 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 927 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 928 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 929 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 930 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 931 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 932 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 933 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 934 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 935 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 936 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 937 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 938 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 939 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 940 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 941 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 942 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 943 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 944 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 945 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 946 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 947 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 948 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 949 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 950 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 951 |
+
[2026-09-21 07:50:34] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 29238.8 | throughput per GPU (TFLOP/s/GPU): 1.2 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.227924E+01 | loss scale: 1.0 | grad norm: 12.617 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 |Number of parameters in transformer block in billions: 17.55
|
| 952 |
+
|
| 953 |
+
Number of parameters in embedding layers in billions: 0.72
|
| 954 |
+
Total number of parameters in billions: 18.28
|
| 955 |
+
Number of parameters in most loaded shard in billions: 18.2753
|
| 956 |
+
Theoretical memory footprints: weight and optimizer=130715.23 MB
|
| 957 |
+
[Rank 0] (after 1 iterations) memory (MB) | allocated: 49136.10400390625 | max allocated: 49136.1044921875 | reserved: 54394.0 | max reserved: 54394.0
|
| 958 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 959 |
+
warnings.warn(
|
| 960 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 961 |
+
warnings.warn(
|
| 962 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 963 |
+
warnings.warn(
|
| 964 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 965 |
+
warnings.warn(
|
| 966 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 967 |
+
warnings.warn(
|
| 968 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 969 |
+
warnings.warn(
|
| 970 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 971 |
+
warnings.warn(
|
| 972 |
+
/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance.
|
| 973 |
+
warnings.warn(
|
| 974 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 975 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 976 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 977 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 978 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 979 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 980 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 981 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 982 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 983 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 984 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 985 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 986 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 987 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 988 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 989 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 990 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 991 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 992 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 993 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 994 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 995 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 996 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 997 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 998 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 999 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1000 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 1001 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 1002 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1003 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1004 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
|
| 1005 |
+
warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
|
| 1006 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1007 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1008 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1009 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1010 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1011 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1012 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1013 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1014 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1015 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1016 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1017 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1018 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1019 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1020 |
+
/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
|
| 1021 |
+
warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
|
| 1022 |
+
[2026-09-21 07:50:34] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 164.0 | throughput per GPU (TFLOP/s/GPU): 215.7 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.227088E+01 | loss scale: 1.0 | grad norm: 12.534 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 |
|
| 1023 |
+
[2026-09-21 07:50:35] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 111.7 | throughput per GPU (TFLOP/s/GPU): 316.7 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.088328E+01 | loss scale: 1.0 | grad norm: 11.023 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 |
|
| 1024 |
+
Capture CUDA graph for training!!!
|
| 1025 |
+
CUDA graph capture done!!!
|
| 1026 |
+
[2026-09-21 07:50:36] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 1248.8 | throughput per GPU (TFLOP/s/GPU): 28.3 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.448181E+01 | loss scale: 1.0 | grad norm: 72.911 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36608 | min safe capacity: 1.1172 |
|
| 1027 |
+
[2026-09-21 07:50:36] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 110.2 | throughput per GPU (TFLOP/s/GPU): 320.9 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.808459E+01 | loss scale: 1.0 | grad norm: 108.268 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1028 |
+
[2026-09-21 07:50:36] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 112.2 | throughput per GPU (TFLOP/s/GPU): 315.3 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.491727E+01 | loss scale: 1.0 | grad norm: 83.473 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1029 |
+
[2026-09-21 07:50:36] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 114.3 | throughput per GPU (TFLOP/s/GPU): 309.5 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.143662E+01 | loss scale: 1.0 | grad norm: 26.627 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1030 |
+
[2026-09-21 07:50:36] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 108.9 | throughput per GPU (TFLOP/s/GPU): 324.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.166943E+01 | loss scale: 1.0 | grad norm: 12.800 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1031 |
+
[2026-09-21 07:50:36] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 109.0 | throughput per GPU (TFLOP/s/GPU): 324.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.131635E+01 | loss scale: 1.0 | grad norm: 10.441 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1032 |
+
[2026-09-21 07:50:36] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 109.1 | throughput per GPU (TFLOP/s/GPU): 324.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.096943E+01 | loss scale: 1.0 | grad norm: 9.147 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1033 |
+
[2026-09-21 07:50:37] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 108.6 | throughput per GPU (TFLOP/s/GPU): 325.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.073911E+01 | loss scale: 1.0 | grad norm: 9.755 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1034 |
+
[2026-09-21 07:50:37] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 108.5 | throughput per GPU (TFLOP/s/GPU): 326.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.048286E+01 | loss scale: 1.0 | grad norm: 8.843 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 |
|
| 1035 |
+
[after training is done] datetime: 2026-09-21 07:50:37
|
| 1036 |
+
W0921 07:50:37.341000 527457 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers
|
| 1037 |
+
W0921 07:50:37.342000 527457 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 527563 closing signal SIGTERM
|
| 1038 |
+
W0921 07:50:37.342000 527457 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 527564 closing signal SIGTERM
|
| 1039 |
+
W0921 07:50:37.343000 527457 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 527565 closing signal SIGTERM
|
| 1040 |
+
W0921 07:50:37.344000 527457 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 527566 closing signal SIGTERM
|
| 1041 |
+
W0921 07:50:37.345000 527457 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 527567 closing signal SIGTERM
|
| 1042 |
+
W0921 07:50:37.346000 527457 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 527568 closing signal SIGTERM
|
| 1043 |
+
W0921 07:50:37.348000 527457 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 527569 closing signal SIGTERM
|
| 1044 |
+
W0921 07:50:37.349000 527457 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 527570 closing signal SIGTERM
|
| 1045 |
+
Traceback (most recent call last):
|
| 1046 |
+
File "/usr/local/bin/torchrun", line 8, in <module>
|
| 1047 |
+
sys.exit(main())
|
| 1048 |
+
^^^^^^
|
| 1049 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
|
| 1050 |
+
return f(*args, **kwargs)
|
| 1051 |
+
^^^^^^^^^^^^^^^^^^
|
| 1052 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main
|
| 1053 |
+
run(args)
|
| 1054 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run
|
| 1055 |
+
elastic_launch(
|
| 1056 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__
|
| 1057 |
+
return launch_agent(self._config, self._entrypoint, list(args))
|
| 1058 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 1059 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent
|
| 1060 |
+
result = agent.run()
|
| 1061 |
+
^^^^^^^^^^^
|
| 1062 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
| 1063 |
+
result = f(*args, **kwargs)
|
| 1064 |
+
^^^^^^^^^^^^^^^^^^
|
| 1065 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run
|
| 1066 |
+
result = self._invoke_run(role)
|
| 1067 |
+
^^^^^^^^^^^^^^^^^^^^^^
|
| 1068 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run
|
| 1069 |
+
time.sleep(monitor_interval)
|
| 1070 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler
|
| 1071 |
+
raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
|
| 1072 |
+
torch.distributed.elastic.multiprocessing.api.SignalException: Process 527457 got signal: 15
|
granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/results.tsv
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir
|
| 2 |
+
KIMI_K2_L2 ultraep matched 48 4096 8 1.17578125 none 1 OK 110.06 108.20 90375 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep1
|
| 3 |
+
KIMI_K2_L2 ultraep matched 48 4096 8 1.17578125 none 2 OK 110.08 108.20 90375 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep2
|
| 4 |
+
KIMI_K2_L2 ultraep matched 48 4096 8 1.17578125 none 3 OK 110.08 108.20 90375 /workspace/granularity_next_b200/tierb_models/attempts/KIMI_K2_L2/ultraep/tierb/tierb_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125/rep3
|