diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/manifest.txt b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/manifest.txt new file mode 100644 index 0000000000000000000000000000000000000000..1c9671d8de911af38e04800530bfa290e683301b --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/manifest.txt @@ -0,0 +1,19 @@ +timestamp: 2026-09-21T07:57:49+00:00 +host: sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0 +gpu: NVIDIA B200 x8 +driver: 580.95.05 +study_scripts_branch: granularity-ultraep +study_scripts_commit: 05dd2cb +campaign: ultraep +backend: ultraep +phase: tierbmem +gbs: 8 +seq: 4096 +configs: G1 G2 G3 G4 +capacity: 1.25 +repeats: 3 +nsys: 1 +megatron_repo: /workspace/Megatron-UltraEP +megatron_branch_required: ultraep-better-memory-dump-support +megatron_branch_actual: ultraep-better-memory-dump-support +megatron_commit: c3a8a8a7 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/pip_freeze.txt b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/pip_freeze.txt new file mode 100644 index 0000000000000000000000000000000000000000..9c69210843de9f3bd23fc067a2c2591a9db6f57e --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/pip_freeze.txt @@ -0,0 +1,317 @@ +absl-py==2.3.0 +aiohappyeyeballs @ file:///rapids/aiohappyeyeballs-2.6.1-py3-none-any.whl#sha256=f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 +aiohttp @ file:///rapids/aiohttp-3.12.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=94f98e0e5a49f89b252e115844f756c04fc8050f38252a32a3dd994ce8121f10 +aiosignal @ file:///rapids/aiosignal-1.3.2-py2.py3-none-any.whl#sha256=45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5 +annotated-types==0.7.0 +anyio==4.9.0 +apex @ file:///opt/pytorch/apex/dist/apex-0.1-cp312-cp312-linux_x86_64.whl#sha256=a1508e4a386362e9c8f46c92ccada9110e421212b964c694235d0fd5a224b170 +argon2-cffi==25.1.0 +argon2-cffi-bindings==21.2.0 +arrow==1.3.0 +asciitree @ file:///rapids/asciitree-0.3.3-py3-none-any.whl#sha256=3e507ccfaeb417823158091fdb9d26b0205f6a59ba680b479a71db65c73d01d4 +asttokens==3.0.0 +astunparse==1.6.3 +async-lru==2.0.5 +attrs==25.3.0 +audioread==3.0.1 +babel==2.17.0 +beautifulsoup4==4.13.4 +black==25.1.0 +bleach==6.2.0 +blis==0.7.11 +cachetools @ file:///rapids/cachetools-6.0.0-py3-none-any.whl#sha256=82e73ba88f7b30228b5507dce1a1f878498fc669d972aef2dde4f3a3c24f103e +catalogue==2.0.10 +certifi==2025.4.26 +cffi==1.17.1 +charset-normalizer==3.4.2 +click==8.5.0 +cloudpathlib==0.21.1 +cloudpickle @ file:///rapids/cloudpickle-3.1.1-py3-none-any.whl#sha256=c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e +cmake==3.31.6 +comm==0.2.2 +confection==0.1.5 +contourpy==1.3.2 +cuda-bindings @ file:///rapids/cuda_bindings-12.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=ff0e28d1e34758654b9c961e1f55e4786e49aee6a4dbceaf3cc24c46c672df7e +cuda-python @ file:///rapids/cuda_python-12.9.0-py3-none-any.whl#sha256=926acba49b2c0a0374c61b7c98f337c085199cf51cdfe4d6423c4129c20547a7 +cudf @ file:///rapids/cudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b779a8e8c0e8577b9b560895958496da211bfac49cdc4f11a49e8d18cb82e348 +cudf-polars @ file:///rapids/cudf_polars-25.4.0-py3-none-any.whl#sha256=227b6c5147e93ea86282c8f7219118f916769fa2ca0cdd931e97a148aa66b869 +cugraph @ file:///rapids/cugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b837c2dc2f7b235c702e4f8df64479b8fcbb342f848516d5faf0cdc4b5f31f29 +cugraph-service-client @ file:///rapids/cugraph_service_client-25.4.0-py3-none-any.whl#sha256=7e568c3e41154b9752f5173555ffcc282ac52d72343feeb79653b2e2b1232470 +cugraph-service-server @ file:///rapids/cugraph_service_server-25.4.0-py3-none-any.whl#sha256=a2830dcffafcccb16e12ebf85b98f8f247869d38c01ab4dd8664a1e7439d31be +cuml @ file:///rapids/cuml-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1e90a464a18b094e24797f1676688c5d7696c4cf97c934504214a53d5aa09aa5 +cupy-cuda12x @ file:///rapids/cupy_cuda12x-13.3.0-cp312-cp312-linux_x86_64.whl#sha256=6a2dfa0aa1e8189a10d1534276c08f1d2f73b747fd28bfbe7b9f4ef66d9af73f +cuvs @ file:///rapids/cuvs-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b12f6b9f80e053a3b38542c2947d2516e468c02cad54a13d8b1a4a61d5aa7914 +cycler==0.12.1 +cymem==2.0.11 +Cython==3.1.1 +dask @ file:///rapids/dask-2025.2.0-py3-none-any.whl#sha256=f0fdeef6ceb0a06569d456c9e704f220f7f54e80f3a6ea42ab98cea6bc642b6e +dask-cuda @ file:///rapids/dask_cuda-25.4.0-py3-none-any.whl#sha256=442d87da0a95d00ca2a262b64c76e785a32ad905c82000fa2227932f81390e19 +dask-cudf @ file:///rapids/dask_cudf-25.4.0-py3-none-any.whl#sha256=5e7cea0b26daea17da6e0bb457435e24460f76ba740358a3a18e239d6015c95c +datasets==5.0.1 +debugpy==1.8.14 +decorator==5.2.1 +-e git+https://github.com/deepseek-ai/DeepEP.git@34152ae28f80bcc3ee38d7a12cb2ad87cfd4ea72#egg=deep_ep +defusedxml==0.7.1 +dill==0.4.1 +distributed @ file:///rapids/distributed-2025.2.0-py3-none-any.whl#sha256=368462084be47092d0a835f6c64c9b852da64dc88302b76d5f93bd5131228c67 +distributed-ucxx @ file:///rapids/distributed_ucxx-0.43.0-py3-none-any.whl#sha256=94f60d0ccb0c94ba8dbfa1eaf455874d18aaab17e2ae218c38590e9fdc2d721d +dm-tree==0.1.9 +einops==0.8.1 +execnet==2.1.1 +executing==2.2.0 +expecttest==0.3.0 +fasteners @ file:///rapids/fasteners-0.19-py3-none-any.whl#sha256=758819cb5d94cdedf4e836988b74de396ceacb8e2794d21f82d131fd9ee77237 +fastjsonschema==2.21.1 +fastrlock @ file:///rapids/fastrlock-0.8.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl#sha256=5f13ec08f1adb1aa916c384b05ecb7dbebb8df9ea81abd045f60941c6283a670 +filelock==3.18.0 +flash_attn @ file:///opt/pytorch/flash_attn-2.7.4.post1-cp312-cp312-linux_x86_64.whl#sha256=a10455d6c56b387785b7413fd0bbcb392b42b3d4f4b88fda5d86cc8a9ce9182a +fonttools==4.58.1 +fqdn==1.5.1 +frozenlist @ file:///rapids/frozenlist-1.6.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=4e1be9111cb6756868ac242b3c2bd1f09d9aea09846e4f5c23715e7afb647103 +fsspec==2025.5.1 +gast==0.6.0 +grpcio==1.62.1 +h11==0.16.0 +hf-xet==1.6.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.32.0 +hypothesis==6.130.8 +idna==3.10 +importlib_metadata==8.7.0 +iniconfig==2.1.0 +intel-openmp==2021.4.0 +ipykernel==6.29.5 +ipython==9.3.0 +ipython_pygments_lexers==1.1.1 +isoduration==20.11.0 +isort==6.0.1 +jedi==0.19.2 +Jinja2==3.1.6 +joblib==1.5.1 +json5==0.12.0 +jsonpointer==3.0.0 +jsonschema==4.24.0 +jsonschema-specifications==2025.4.1 +jupyter-events==0.12.0 +jupyter-lsp==2.2.5 +jupyter_client==8.6.3 +jupyter_core==5.8.1 +jupyter_server==2.16.0 +jupyter_server_terminals==0.5.3 +jupyterlab==4.4.3 +jupyterlab_code_formatter==3.0.2 +jupyterlab_pygments==0.3.0 +jupyterlab_server==2.27.3 +jupyterlab_tensorboard_pro @ file:///builder/jupyterlab_tensorboard_pro-4.0.0-py2.py3-none-any.whl#sha256=6bfcb5ced3358eb72150f4b2ab91312faef2f0ca14d845e8379d8fa52df6902e +jupytext==1.17.2 +kiwisolver==1.4.8 +kvikio @ file:///rapids/kvikio-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=8b40fd8ea9e4a2974c04b50b7fc26b06094ac807f9a233281d0bafbaa71b1d85 +langcodes==3.5.0 +language_data==1.3.0 +lazy_loader==0.4 +libcudf @ file:///rapids/libcudf-25.4.0-py3-none-linux_x86_64.whl#sha256=238320b4958c1737bba2a609673e2539a78216b811b5498a68e00395d9b021a4 +libcugraph @ file:///rapids/libcugraph-25.4.0-py3-none-linux_x86_64.whl#sha256=29085e901b2f7569ff2653bb1355fe6fad20f2fcaa585f5649da9867d8df766d +libcuml @ file:///rapids/libcuml-25.4.0-py3-none-linux_x86_64.whl#sha256=8e2e444c72b0e995e15a48da37ff12fb410a47a2baa69ba86ccc9b1a3db91e8c +libcuvs @ file:///rapids/libcuvs-25.4.0-py3-none-linux_x86_64.whl#sha256=e2f8db61c92b769c590dd752f43439ddd62b2f6592a3c337362b6c1ef220c3b9 +libkvikio @ file:///rapids/libkvikio-25.4.0-py3-none-linux_x86_64.whl#sha256=da8a5713b05fc478bf40529a3e0fb61a2d605bb5a70ca4d461f96a6005dbe8b7 +libraft @ file:///rapids/libraft-25.4.0-py3-none-linux_x86_64.whl#sha256=b921dec005aa4a321dd156776a18376feb47c96a5b67eec125c8a8e19edd1e24 +librmm @ file:///rapids/librmm-25.4.0-py3-none-linux_x86_64.whl#sha256=ff951811bda729d4fa1109ab906225ae1f489a6ec2eeab5d1b674cc34bd4422f +librmm-cu12 @ file:///rapids/librmm_cu12-25.4.0-py3-none-any.whl#sha256=29296ef2b0e90679a0db280e86015be12d6ab9f4e62a181aa085f62dbadd06b3 +librosa==0.11.0 +libucxx @ file:///rapids/libucxx-0.43.0-py3-none-linux_x86_64.whl#sha256=f15849d7ac2fb68f0fd8998838a13321ddcfb0c594f7e6cbc401a3bcc4750566 +lightning-thunder @ file:///opt/pytorch/lightning-thunder +lightning-utilities==0.14.3 +lintrunner==0.12.7 +llvmlite==0.42.0 +locket @ file:///rapids/locket-1.0.0-py2.py3-none-any.whl#sha256=b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3 +looseversion==1.3.0 +marisa-trie==1.2.1 +Markdown==3.8 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +matplotlib==3.10.3 +matplotlib-inline==0.1.7 +mdit-py-plugins==0.4.2 +mdurl==0.1.2 +-e git+ssh://git@github.com/sriram-17-17/my-Megatron-MoE-Echo.git@b7c52442d011da6743b3206aac1b06b8b122dc39#egg=megatron_core +mistune==3.1.3 +mkl==2021.1.1 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +ml_dtypes==0.6.0 +mock==5.2.0 +-e git+ssh://git@github.com/sriram-17-17/MoonEP.git@d8bc6667fad7177e6e09b637d773fa67f08bae93#egg=moonep +mpmath==1.3.0 +msgpack==1.1.0 +multidict @ file:///rapids/multidict-6.4.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=a920f9cf2abdf6e493c519492d892c362007f113c94da4c239ae88429835bad1 +multiprocess==0.70.19 +murmurhash==1.0.13 +mypy_extensions==1.1.0 +nbclient==0.10.2 +nbconvert==7.16.6 +nbformat==5.10.4 +nest-asyncio==1.6.0 +networkx==3.5 +ninja==1.11.1.4 +notebook==7.4.3 +notebook_shim==0.2.4 +numba==0.59.1 +numba-cuda @ file:///rapids/numba_cuda-0.4.0-py3-none-any.whl#sha256=5b95a5f42068771e37142944d44fe844fa87f1b2f89051461a4cb0af91bcc28a +numcodecs @ file:///rapids/numcodecs-0.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=da2230484e6102e5fa3cc1a5dd37ca1f92dfbd183d91662074d6f7574e3e8f53 +numpy==1.26.4 +nvdlfw_inspect @ git+https://github.com/NVIDIA/nvidia-dlfw-inspect.git@6b60eb0e675606fb2fbbfcb12667ecbec75eaf87 +nvfuser==0.2.27a0+9bf5aca +nvidia-cuda-cccl-cu12==12.9.27 +nvidia-cudnn-frontend @ file:///opt/pytorch/lightning-thunder/tmp/cudnn_frontend +nvidia-cutlass-dsl==4.4.2 +nvidia-cutlass-dsl-libs-base==4.4.2 +nvidia-dali-cuda120==1.50.0 +nvidia-ml-py @ file:///rapids/nvidia_ml_py-12.575.51-py3-none-any.whl#sha256=eb8641800d98ce40a22f479873f34b482e214a7e80349c63be51c3919845446e +nvidia-modelopt==0.29.0 +nvidia-modelopt-core==0.29.0 +nvidia-nvcomp-cu12==4.2.0.14 +nvidia-nvimgcodec-cu12==0.5.0.13 +nvidia-nvjpeg-cu12==12.4.0.16 +nvidia-nvjpeg2k-cu12==0.8.1.40 +nvidia-nvshmem-cu12==3.7.2 +nvidia-nvtiff-cu12==0.5.0.67 +nvidia-resiliency-ext==0.4.0 +nvtx @ file:///rapids/nvtx-0.2.11-cp312-cp312-linux_x86_64.whl#sha256=73521e37cba42b4db0b3cd68fc5eb814e55c500a61eeed6edcdf6746315ac53e +nx-cugraph @ file:///rapids/nx_cugraph-25.4.0-py3-none-any.whl#sha256=e625444486cc899a77bd48e7da76a2edd95c0006dd95fbf323745feae7f33111 +onnx @ file:///opt/pytorch/pytorch/third_party/onnx +onnx-ir==1.0.0 +onnxscript==0.7.2 +opt_einsum==3.4.0 +optree==0.16.0 +overrides==7.7.0 +packaging==23.2 +pandas @ file:///rapids/pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319 +pandocfilters==1.5.1 +parso==0.8.4 +partd @ file:///rapids/partd-1.4.2-py3-none-any.whl#sha256=978e4ac767ec4ba5b86c6eaa52e5a2a3bc748a2ca839e8cc798f1cc6ce6efb0f +pathspec==0.12.1 +pexpect==4.9.0 +pillow==11.2.1 +platformdirs==4.3.8 +pluggy==1.6.0 +ply @ file:///rapids/ply-3.11-py2.py3-none-any.whl#sha256=096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce +polars @ file:///rapids/polars-1.25.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=f7fcbb4f476784384ccda48757fca4e8c2e2c5a0a3aef3717aaf56aee4e30e09 +polygraphy==0.49.20 +pooch==1.8.2 +preshed==3.0.10 +prometheus_client==0.22.1 +prompt_toolkit==3.0.51 +propcache @ file:///rapids/propcache-0.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=563f9d8c03ad645597b8d010ef4e9eab359faeb11a0a2ac9f7b4bc8c28ebef25 +protobuf==4.24.4 +psutil==7.0.0 +ptyprocess==0.7.0 +PuLP==3.2.1 +pure_eval==0.2.3 +pyarrow==25.0.1 +pybind11==2.13.6 +pybind11_global==2.13.6 +pycocotools @ git+https://github.com/nvidia/cocoapi.git@08350a157b41f0755a536441b5abec89ab934c5b#subdirectory=PythonAPI +pycparser==2.22 +pydantic==2.11.5 +pydantic_core==2.33.2 +Pygments==2.19.1 +pylibcudf @ file:///rapids/pylibcudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=401b8bfd022d06f84e120779ca7c3d01a28b68762aaa8f7873b10d6b1975cf6a +pylibcugraph @ file:///rapids/pylibcugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=061c7389a88c815b54fd5d3a3584fb87bf693e5602f5be335260d0ba575ba557 +pylibcugraphops @ file:///rapids/pylibcugraphops-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=a25ef4b22166c80718d5c7e5bddb755e81a2fe340fa5becd0ac36ec3edfb208a +pylibraft @ file:///rapids/pylibraft-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=f2ecd23fe51f9c2caa87bbbe85cb32fcaff4ecfb52ceb9a9991bc50f82782c2e +pylibwholegraph @ file:///rapids/pylibwholegraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b82913d1e38bda76d8c272fbd8cc9072d987209d3d47cc93a464a37c2925aca2 +pynvjitlink @ file:///rapids/pynvjitlink-0.3.0-cp312-cp312-linux_x86_64.whl#sha256=066caf1314730643d3b48c9847bd65789defdd6a7c78993176eb3d8bbe3f4919 +pynvml @ file:///rapids/pynvml-12.0.0-py3-none-any.whl#sha256=fdff84b62a27dbe98e08e1a647eb77342bef1aebe0878bcd15e99a83fcbecb9e +pyparsing==3.2.3 +pytest==8.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==15.1 +pytest-shard==0.1.2 +pytest-xdist==3.7.0 +python-dateutil==2.9.0.post0 +python-hostlist==2.2.1 +python-json-logger==3.3.0 +pytorch-triton @ file:///tmp/dist/pytorch_triton-3.3.0%2Bgit96316ce52.nvinternal-cp312-cp312-linux_x86_64.whl#sha256=6d6921772b9dfd0243c055e0d054de42a113e92cabcfdd1d3e80a5ad9c08cf7d +pytz @ file:///rapids/pytz-2023.4-py2.py3-none-any.whl#sha256=f90ef520d95e7c46951105338d918664ebfd6f1d995bd7d153127ce90efafa6a +PyYAML==6.0.2 +pyzmq==26.4.0 +raft-dask @ file:///rapids/raft_dask-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=36cf37ba6ce35454c4729c2cb597a360f22780eefb11d514a16a07137da5eb6c +rapids-dask-dependency @ file:///rapids/rapids_dask_dependency-25.4.0a0-py3-none-any.whl#sha256=484faca626e7760112a7d8ad3426f5bdd720b609006d8dca87c2607076a348bd +rapids-logger @ file:///rapids/rapids_logger-0.1.18-py3-none-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=3617b2d86291ac6cb957542895f20fc9737ace29773ca90337e6fb04c73bd120 +referencing==0.36.2 +regex==2024.11.6 +requests==2.32.3 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rich==14.0.0 +rmm @ file:///rapids/rmm-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1824bcab3836296666980a997ddcb9c4bca24dfb873012cfa0a7ecea583ca712 +rpds-py==0.25.1 +safetensors==0.5.3 +scikit-learn==1.6.1 +scipy==1.15.3 +Send2Trash==1.8.3 +setuptools==78.1.1 +shellingham==1.5.4 +six==1.16.0 +smart-open==7.1.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.13.1 +soupsieve==2.7 +soxr==0.5.0.post1 +spacy==3.7.5 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +srsly==2.5.1 +stack-data==0.6.3 +sympy==1.14.0 +tabulate==0.9.0 +tbb==2021.13.1 +tblib @ file:///rapids/tblib-3.1.0-py3-none-any.whl#sha256=670bb4582578134b3d81a84afa1b016128b429f3d48e6cbbaecc9d15675e984e +tensorboard==2.16.2 +tensorboard-data-server==0.7.2 +tensorrt @ file:///workspace/TensorRT-10.11.0.33/python/tensorrt-10.11.0.33-cp312-none-linux_x86_64.whl#sha256=c2f18f72758a3d54a71901495c8fda9e7094a8bcdd2a5b6b58967fbf05733b3a +terminado==0.18.1 +thinc==8.2.5 +threadpoolctl==3.6.0 +thriftpy2 @ file:///rapids/thriftpy2-0.5.2-cp312-cp312-linux_x86_64.whl#sha256=c0c7d939bdc5ccf7739154959938445d5fdf72f1d6e12bd563947a36f67e2147 +tinycss2==1.4.0 +toolz @ file:///rapids/toolz-1.0.0-py3-none-any.whl#sha256=292c8f1c4e7516bf9086f8850935c799a874039c8bcf959d47b600e4c44a6236 +torch @ file:///opt/transfer/torch-2.8.0a0%2B5228986c39.nv25.6-29946474-cp312-cp312-linux_x86_64.whl#sha256=7a3c9ff5288d8102a31296b3d62491f3f47c8820a5c9aaefc7dc0fe20e1e8ad5 +torch_tensorrt @ file:///opt/pytorch/torch_tensorrt/dist/torch_tensorrt-2.8.0a0-cp312-cp312-linux_x86_64.whl#sha256=26f4d7a04e7e6e017a5b917b8b98b83720bb2bb0195f48d25f52d567d286bfe1 +torchao @ file:///opt/pytorch/ao +torchprofile==0.0.4 +torchvision @ file:///opt/transfer/torchvision-0.22.0a0%2B95f10a4e-29946474-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=746a3c674cc61680a86f4a14b374b21af1b5c21f629c42765b40c9494f6b234d +tornado==6.5.1 +tqdm==4.67.1 +traitlets==5.14.3 +transformer_engine @ file:///pvcdata/aiinfra/b-aghosh/wheels/transformer_engine-2.10.0.dev0%2B10f6128d-cp312-cp312-linux_x86_64.whl#sha256=35cf5f4b83b2baad864e12e512bb422e0929409ac05610812fa521d4942d0189 +treelite @ file:///rapids/treelite-4.4.1-py3-none-linux_x86_64.whl#sha256=174741724205540a1a5b37da81483aeff55d89548f42b3461cf07acac7ae96eb +typer==0.16.0 +types-dataclasses==0.6.6 +types-python-dateutil==2.9.0.20250516 +typing-inspection==0.4.1 +typing_extensions==4.14.0 +tzdata @ file:///rapids/tzdata-2025.2-py2.py3-none-any.whl#sha256=1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 +ucx-py @ file:///rapids/ucx_py-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=4a452fd8d8c983283ada3d49086e3657dc2ee94fbad08efdd708d15ab6e0b73a +ucxx @ file:///rapids/ucxx-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=f1b4e55cb45c66fe88a2d8f9c2b87751165f3cfec21fbdc9c47c4ddff75e8a5f +-e git+https://github.com/Dots-Infra/UltraEP.git@94cab099b44fffa99a82fea99e7c12d89cf65e4f#egg=ultra_ep +uri-template==1.3.0 +urllib3==2.0.7 +wasabi==1.1.3 +wcwidth==0.2.13 +weasel==0.4.1 +webcolors==24.11.1 +webencodings==0.5.1 +websocket-client==1.8.0 +Werkzeug==3.1.3 +wheel==0.45.1 +wrapt==1.17.2 +xdoctest==1.0.2 +xgboost @ file:///rapids/xgboost-2.1.4-py3-none-manylinux_2_39_x86_64.whl#sha256=25eeca27f4048fc491940b3a0b4b5782ee48c33cb3aec059cef71f8da542930b +xxhash==4.0.1 +yarl @ file:///rapids/yarl-1.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=bf099e2432131093cc611623e0b0bcc399b8cddd9a91eded8bfb50402ec35018 +zarr @ file:///rapids/zarr-2.18.7-py3-none-any.whl#sha256=ac3dc4033e9ae4e9d7b5e27c97ea3eaf1003cc0a07f010bd83d5134bf8c4b223 +zict @ file:///rapids/zict-3.0.0-py2.py3-none-any.whl#sha256=5796e36bd0e0cc8cf0fbc1ace6a68912611c1dbd74750a3f3026b9b9d6a327ae +zipp==3.22.0 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/provenance.txt b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/provenance.txt new file mode 100644 index 0000000000000000000000000000000000000000..01fc8070bfd194c13254fe049f10460ceeb7cc9f --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/provenance.txt @@ -0,0 +1,90 @@ +============================================================== +PROVENANCE: ultraep__tierbmem__gbs8 +============================================================== +captured_at : 2026-09-21T07:57:34+00:00 +hostname : sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0 +kernel : 6.8.0-1043-nvidia +cwd : /workspace/blackwell_run_scripts +command : run_ladder.sh --phase tierbmem --gbs 8 --out /workspace/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem --nsys + +---- GPU / driver -------------------------------------------- + 0, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 1, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 2, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 3, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 4, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 5, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 6, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 7, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + persistence/clocks: + 0, 120 MHz, 1965 MHz, 1000.00 W, 29 + 1, 1965 MHz, 1965 MHz, 1000.00 W, 35 + 2, 120 MHz, 1965 MHz, 1000.00 W, 34 + 3, 120 MHz, 1965 MHz, 1000.00 W, 30 + 4, 1965 MHz, 1965 MHz, 1000.00 W, 30 + 5, 120 MHz, 1965 MHz, 1000.00 W, 34 + 6, 120 MHz, 1965 MHz, 1000.00 W, 33 + 7, 120 MHz, 1965 MHz, 1000.00 W, 29 + + NVLink topology: + GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 CPU Affinity NUMA Affinity GPU NUMA ID + GPU0 X NV18 NV18 NV18 NV18 NV18 NV18 NV18 PIX NODE SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU1 NV18 X NV18 NV18 NV18 NV18 NV18 NV18 NODE PIX SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU2 NV18 NV18 X NV18 NV18 NV18 NV18 NV18 SYS SYS PIX NODE SYS SYS SYS SYS SYS 28-55 1 N/A + GPU3 NV18 NV18 NV18 X NV18 NV18 NV18 NV18 SYS SYS NODE PIX SYS SYS SYS SYS SYS 28-55 1 N/A + GPU4 NV18 NV18 NV18 NV18 X NV18 NV18 NV18 SYS SYS SYS SYS PIX NODE SYS SYS SYS 56-83 2 N/A + GPU5 NV18 NV18 NV18 NV18 NV18 X NV18 NV18 SYS SYS SYS SYS NODE PIX SYS SYS SYS 56-83 2 N/A + GPU6 NV18 NV18 NV18 NV18 NV18 NV18 X NV18 SYS SYS SYS SYS SYS SYS PIX NODE SYS 84-111 3 N/A + GPU7 NV18 NV18 NV18 NV18 NV18 NV18 NV18 X SYS SYS SYS SYS SYS SYS NODE PIX SYS 84-111 3 N/A + NIC0 PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS SYS NODE + NIC1 NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS SYS NODE + NIC2 SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS + NIC3 SYS SYS NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS + NIC4 SYS SYS SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS + unavailable + +---- toolchain ----------------------------------------------- + nvcc : release 12.9, V12.9.86 + gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 + python : 3.12.3 + torch : 2.8.0a0+5228986c39.nv25.06 cuda=12.9 nccl=2.27.3 + torch arch : ['sm_75', 'sm_80', 'sm_86', 'sm_90', 'sm_100', 'sm_120', 'compute_120'] + transformer_engine: 2.10.0.dev0+10f6128d [/usr/local/lib/python3.12/dist-packages/transformer_engine/__init__.py] + deep_ep : 1.2.1+34152ae [/workspace/DeepEP/deep_ep/__init__.py] + ultra_ep : 1.0.0 [/workspace/UltraEP/ultra_ep/__init__.py] + moonep : 0.0.1 [/workspace/MoonEP/moonep/__init__.py] + megatron : 0.14.0rc5 [None] + flash_attn : 2.7.4.post1 [/usr/local/lib/python3.12/dist-packages/flash_attn/__init__.py] + numpy : 1.26.4 [/usr/local/lib/python3.12/dist-packages/numpy/__init__.py] + huggingface_hub: 1.32.0 [/usr/local/lib/python3.12/dist-packages/huggingface_hub/__init__.py] + +---- repositories -------------------------------------------- + Megatron-MoE-Echo better-memory-dump-support b7c52442d011 + Megatron-UltraEP ultraep-better-memory-dump-support c3a8a8a7b69d + UltraEP main 94cab099b44f + DeepEP HEAD 34152ae28f80 (+3 untracked: build output) + MoonEP megatron-ddp-symmetric-params d8bc6667fad7 + blackwell_run_scripts granularity-ultraep 05dd2cbfb1eb + +---- environment --------------------------------------------- + CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0 + CUDA_DRIVER_VERSION=575.57.08 + CUDA_HOME=/usr/local/cuda + CUDA_MODULE_LOADING=LAZY + CUDA_VERSION=12.9.1.010 + HF_REMOTE_ROOT=granularity_next_b200/tierb_models/memory/KIMI_K2_L2 + NCCL_VERSION=2.27.3 + NVSHMEM_VERSION=3.2.5 + PYTORCH_BUILD_NUMBER=0 + PYTORCH_BUILD_VERSION=2.8.0a0+5228986 + PYTORCH_HOME=/opt/pytorch/pytorch + PYTORCH_VERSION=2.8.0a0+5228986 + TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1 + TORCH_CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0+PTX + TORCH_NCCL_USE_COMM_NONBLOCKING=0 + +---- shared library resolution ------------------------------- + libnvshmem_host.so: not in ldconfig + libnccl.so: /lib/x86_64-linux-gnu/libnccl.so.2 + lrwxrwxrwx 1 root root 20 Sep 21 04:57 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so -> libnvshmem_host.so.3 + -rw-r--r-- 1 root root 179045744 Sep 21 04:54 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so.3 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/mem_samples.csv b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..b91df61ce8f368377cbdb03bca4e11d99a8fada2 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/mem_samples.csv @@ -0,0 +1,200 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 725 +1, 725 +2, 725 +3, 725 +4, 725 +5, 725 +6, 725 +7, 725 +0, 1007 +1, 1071 +2, 1023 +3, 1029 +4, 1041 +5, 1037 +6, 1051 +7, 1051 +0, 1927 +1, 1927 +2, 1927 +3, 1927 +4, 1927 +5, 1927 +6, 1927 +7, 1927 +0, 3759 +1, 3775 +2, 3775 +3, 3771 +4, 3775 +5, 3813 +6, 3799 +7, 3777 +0, 50559 +1, 50559 +2, 50559 +3, 50559 +4, 50559 +5, 50559 +6, 50559 +7, 50559 +0, 55555 +1, 55555 +2, 55555 +3, 55555 +4, 55555 +5, 55555 +6, 55555 +7, 55555 +0, 60691 +1, 60693 +2, 60691 +3, 60691 +4, 60691 +5, 60691 +6, 60691 +7, 60691 +0, 61267 +1, 61269 +2, 61267 +3, 61267 +4, 61267 +5, 61267 +6, 61267 +7, 61267 +0, 61267 +1, 61269 +2, 61267 +3, 61267 +4, 61267 +5, 61267 +6, 61267 +7, 61267 +0, 64465 +1, 64467 +2, 64465 +3, 64465 +4, 64465 +5, 64465 +6, 64465 +7, 64465 +0, 64465 +1, 64467 +2, 64465 +3, 64465 +4, 64465 +5, 64465 +6, 64465 +7, 64465 +0, 65963 +1, 65965 +2, 65963 +3, 65963 +4, 65963 +5, 65963 +6, 65963 +7, 65963 +0, 65963 +1, 65965 +2, 65963 +3, 65963 +4, 65963 +5, 65963 +6, 65963 +7, 65963 +0, 65969 +1, 65969 +2, 65969 +3, 65969 +4, 65969 +5, 65969 +6, 65969 +7, 65969 +0, 68271 +1, 68259 +2, 68293 +3, 68257 +4, 68321 +5, 68315 +6, 68311 +7, 68295 +0, 69055 +1, 69055 +2, 69055 +3, 69055 +4, 69055 +5, 69055 +6, 69055 +7, 69055 +0, 75773 +1, 75789 +2, 75759 +3, 75751 +4, 75771 +5, 75805 +6, 75789 +7, 75743 +0, 90291 +1, 90297 +2, 90291 +3, 90347 +4, 90347 +5, 90347 +6, 90375 +7, 90347 +0, 89593 +1, 15842 +2, 89715 +3, 89735 +4, 89771 +5, 89515 +6, 89427 +7, 89771 +0, 89270 +1, 15455 +2, 13825 +3, 13825 +4, 13825 +5, 13825 +6, 13825 +7, 14151 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260921T075846268578Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260921T075846268578Z.json new file mode 100644 index 0000000000000000000000000000000000000000..42254ffe2b92931a9c60f44fff330433ad750b90 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260921T075846268578Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51522939392, + "active_bytes_current_delta": 21495492608, + "active_bytes_peak": 51522942464, + "active_bytes_peak_delta": 21436775424, + "after": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 51522942464, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 51522942464, + "device": 0, + "inactive_split_bytes_current": 664688128, + "inactive_split_bytes_peak": 2983869440, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 57036242944, + "reserved_bytes_peak": 57036242944, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-09-21T07:58:46.268100Z", + "world_size": 8 + }, + "allocated_bytes_current": 51522939392, + "allocated_bytes_current_delta": 21495492608, + "allocated_bytes_peak": 51522942464, + "allocated_bytes_peak_delta": 21436775424, + "before": { + "active_bytes_current": 30027446784, + "active_bytes_peak": 30086167040, + "allocated_bytes_current": 30027446784, + "allocated_bytes_peak": 30086167040, + "device": 0, + "inactive_split_bytes_current": 175736320, + "inactive_split_bytes_peak": 762938880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 35888562176, + "reserved_bytes_peak": 35888562176, + "segment_count_current": 243, + "segment_count_peak": 243, + "timestamp_utc": "2026-09-21T07:58:17.714991Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 664688128, + "inactive_split_bytes_current_delta": 488951808, + "inactive_split_bytes_peak": 2983869440, + "inactive_split_bytes_peak_delta": 2220930560, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 57036242944, + "reserved_bytes_current_delta": 21147680768, + "reserved_bytes_peak": 57036242944, + "reserved_bytes_peak_delta": 21147680768, + "segment_count_current": 629, + "segment_count_current_delta": 386, + "segment_count_peak": 629, + "segment_count_peak_delta": 386, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260921T075846268578Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.282486Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260921T075846268578Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260921T075846268578Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b876cf3c888d1332a2fb375bb5580ab653e6e41b --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260921T075846268578Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc7842a3b1b5a4941ee87695f04c277bd91ead6398bb4a3e792504298ff9892a +size 715875 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260921T075846454341Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260921T075846454341Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e5de6b9d7a4f6f061bdae0dfb47dda4a37bc16f4 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260921T075846454341Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51522939392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68273843712, + "active_bytes_peak_delta": 16750901248, + "after": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 68273843712, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 68273843712, + "device": 0, + "inactive_split_bytes_current": 664688128, + "inactive_split_bytes_peak": 4417034240, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 701, + "segment_count_peak": 701, + "timestamp_utc": "2026-09-21T07:58:46.453936Z", + "world_size": 8 + }, + "allocated_bytes_current": 51522939392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68273843712, + "allocated_bytes_peak_delta": 16750901248, + "before": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 51522942464, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 51522942464, + "device": 0, + "inactive_split_bytes_current": 664688128, + "inactive_split_bytes_peak": 2983869440, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 57036242944, + "reserved_bytes_peak": 57036242944, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-09-21T07:58:46.284599Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 664688128, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4417034240, + "inactive_split_bytes_peak_delta": 1433164800, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 71468843008, + "reserved_bytes_current_delta": 14432600064, + "reserved_bytes_peak": 71468843008, + "reserved_bytes_peak_delta": 14432600064, + "segment_count_current": 701, + "segment_count_current_delta": 72, + "segment_count_peak": 701, + "segment_count_peak_delta": 72, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260921T075846454341Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.464359Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260921T075846454341Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260921T075846454341Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d85206326cddf13d2ccf67d02ef71409156639bb --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260921T075846454341Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f9a892031729d9f999e8beb845a9304e0e09c3fc36b7362b40af815651c279e +size 570760 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260921T075846579237Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260921T075846579237Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bfb84402a9488c847a0367d42aaf72ddb68f2d8a --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260921T075846579237Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51522939392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68273843712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 68273843712, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 68273843712, + "device": 0, + "inactive_split_bytes_current": 664688128, + "inactive_split_bytes_peak": 4973565952, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 701, + "segment_count_peak": 701, + "timestamp_utc": "2026-09-21T07:58:46.578836Z", + "world_size": 8 + }, + "allocated_bytes_current": 51522939392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68273843712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 68273843712, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 68273843712, + "device": 0, + "inactive_split_bytes_current": 664688128, + "inactive_split_bytes_peak": 4417034240, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 701, + "segment_count_peak": 701, + "timestamp_utc": "2026-09-21T07:58:46.465885Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 664688128, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4973565952, + "inactive_split_bytes_peak_delta": 556531712, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 71468843008, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71468843008, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 701, + "segment_count_current_delta": 0, + "segment_count_peak": 701, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260921T075846579237Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.589307Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260921T075846579237Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260921T075846579237Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4b9aa8f4aa26568724c32c5461928de4098b8150 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260921T075846579237Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bce21d6a3f111883de9e8f3c8cb56e913d7c8c5fb70385be110e7ae16dae31e +size 571946 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260921T075847870881Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260921T075847870881Z.json new file mode 100644 index 0000000000000000000000000000000000000000..492367bf0ec36432076e34a3195ee2ed728433ea --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260921T075847870881Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51539982848, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 68279320576, + "active_bytes_peak_delta": 5476864, + "after": { + "active_bytes_current": 51539982848, + "active_bytes_peak": 68279320576, + "allocated_bytes_current": 51539982848, + "allocated_bytes_peak": 68279320576, + "device": 0, + "inactive_split_bytes_current": 2832877056, + "inactive_split_bytes_peak": 4974112256, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 72043462656, + "reserved_bytes_peak": 72043462656, + "segment_count_current": 724, + "segment_count_peak": 724, + "timestamp_utc": "2026-09-21T07:58:47.870425Z", + "world_size": 8 + }, + "allocated_bytes_current": 51539982848, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 68279320576, + "allocated_bytes_peak_delta": 5476864, + "before": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 68273843712, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 68273843712, + "device": 0, + "inactive_split_bytes_current": 664688128, + "inactive_split_bytes_peak": 4973565952, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 701, + "segment_count_peak": 701, + "timestamp_utc": "2026-09-21T07:58:46.590796Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2832877056, + "inactive_split_bytes_current_delta": 2168188928, + "inactive_split_bytes_peak": 4974112256, + "inactive_split_bytes_peak_delta": 546304, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 72043462656, + "reserved_bytes_current_delta": 574619648, + "reserved_bytes_peak": 72043462656, + "reserved_bytes_peak_delta": 574619648, + "segment_count_current": 724, + "segment_count_current_delta": 23, + "segment_count_peak": 724, + "segment_count_peak_delta": 23, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260921T075847870881Z.pickle", + "timestamp_utc": "2026-09-21T07:58:47.883571Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260921T075847870881Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260921T075847870881Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3911f332f54ff578f3e1693249cb9f700fe12bc7 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260921T075847870881Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4db447a7229380df168531db90f981522697c18dfda764606daf3ac4b8d217c1 +size 842842 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260921T075847997448Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260921T075847997448Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cd14383d1ec682d3490b8186086519a59ea730d6 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260921T075847997448Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51539982848, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68279320576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51539982848, + "active_bytes_peak": 68279320576, + "allocated_bytes_current": 51539982848, + "allocated_bytes_peak": 68279320576, + "device": 0, + "inactive_split_bytes_current": 2832877056, + "inactive_split_bytes_peak": 4974112256, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 72043462656, + "reserved_bytes_peak": 72043462656, + "segment_count_current": 724, + "segment_count_peak": 724, + "timestamp_utc": "2026-09-21T07:58:47.997059Z", + "world_size": 8 + }, + "allocated_bytes_current": 51539982848, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68279320576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51539982848, + "active_bytes_peak": 68279320576, + "allocated_bytes_current": 51539982848, + "allocated_bytes_peak": 68279320576, + "device": 0, + "inactive_split_bytes_current": 2832877056, + "inactive_split_bytes_peak": 4974112256, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 72043462656, + "reserved_bytes_peak": 72043462656, + "segment_count_current": 724, + "segment_count_peak": 724, + "timestamp_utc": "2026-09-21T07:58:47.885224Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2832877056, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4974112256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 72043462656, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 72043462656, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 724, + "segment_count_current_delta": 0, + "segment_count_peak": 724, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260921T075847997448Z.pickle", + "timestamp_utc": "2026-09-21T07:58:48.002146Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260921T075847997448Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260921T075847997448Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f335638cb1e2b9f84f014fc8220a6ac1beccd369 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260921T075847997448Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9f4b39ef2f83aaceed3be6dbabc07345b6b697483802f018ffdcda6735385ff +size 247842 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260921T075848116088Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260921T075848116088Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2cbd0167dc50a95abce49db14f93622846f90574 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260921T075848116088Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51539982848, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68279320576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51539982848, + "active_bytes_peak": 68279320576, + "allocated_bytes_current": 51539982848, + "allocated_bytes_peak": 68279320576, + "device": 0, + "inactive_split_bytes_current": 2832877056, + "inactive_split_bytes_peak": 4974112256, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 72043462656, + "reserved_bytes_peak": 72043462656, + "segment_count_current": 724, + "segment_count_peak": 724, + "timestamp_utc": "2026-09-21T07:58:48.115722Z", + "world_size": 8 + }, + "allocated_bytes_current": 51539982848, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68279320576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51539982848, + "active_bytes_peak": 68279320576, + "allocated_bytes_current": 51539982848, + "allocated_bytes_peak": 68279320576, + "device": 0, + "inactive_split_bytes_current": 2832877056, + "inactive_split_bytes_peak": 4974112256, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 72043462656, + "reserved_bytes_peak": 72043462656, + "segment_count_current": 724, + "segment_count_peak": 724, + "timestamp_utc": "2026-09-21T07:58:48.003020Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2832877056, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4974112256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 72043462656, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 72043462656, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 724, + "segment_count_current_delta": 0, + "segment_count_peak": 724, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260921T075848116088Z.pickle", + "timestamp_utc": "2026-09-21T07:58:48.120598Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260921T075848116088Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260921T075848116088Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..091e8ab4b3bec10bbce31a621c433c7f99732a0c --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260921T075848116088Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a6a3f228eb44ce93a4f8d79109fe61832cd5f9abe52b8cb975ab678b3979da7 +size 247873 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260921T075846268481Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260921T075846268481Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ca237acd7553f179d70048374065d2712997dbad --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260921T075846268481Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51523350016, + "active_bytes_current_delta": 21495903232, + "active_bytes_peak": 51523353088, + "active_bytes_peak_delta": 21437186048, + "after": { + "active_bytes_current": 51523350016, + "active_bytes_peak": 51523353088, + "allocated_bytes_current": 51523350016, + "allocated_bytes_peak": 51523353088, + "device": 1, + "inactive_split_bytes_current": 670568960, + "inactive_split_bytes_peak": 2988295680, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 57036242944, + "reserved_bytes_peak": 57036242944, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-09-21T07:58:46.267921Z", + "world_size": 8 + }, + "allocated_bytes_current": 51523350016, + "allocated_bytes_current_delta": 21495903232, + "allocated_bytes_peak": 51523353088, + "allocated_bytes_peak_delta": 21437186048, + "before": { + "active_bytes_current": 30027446784, + "active_bytes_peak": 30086167040, + "allocated_bytes_current": 30027446784, + "allocated_bytes_peak": 30086167040, + "device": 1, + "inactive_split_bytes_current": 175736320, + "inactive_split_bytes_peak": 762938880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 35888562176, + "reserved_bytes_peak": 35888562176, + "segment_count_current": 243, + "segment_count_peak": 243, + "timestamp_utc": "2026-09-21T07:58:17.714854Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 670568960, + "inactive_split_bytes_current_delta": 494832640, + "inactive_split_bytes_peak": 2988295680, + "inactive_split_bytes_peak_delta": 2225356800, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 57036242944, + "reserved_bytes_current_delta": 21147680768, + "reserved_bytes_peak": 57036242944, + "reserved_bytes_peak_delta": 21147680768, + "segment_count_current": 629, + "segment_count_current_delta": 386, + "segment_count_peak": 629, + "segment_count_peak_delta": 386, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260921T075846268481Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.282473Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260921T075846268481Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260921T075846268481Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d6eba603c03853d57d6689587dc4d96a15415df8 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260921T075846268481Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:677e6d8fe5e16ad464ca83884fdb18780d747182c9963aefc2baca68cb4e5633 +size 715434 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260921T075846454221Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260921T075846454221Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a38e42f730d9abc476ad6aeeafb6266e2f697892 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260921T075846454221Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51523350016, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68273233408, + "active_bytes_peak_delta": 16749880320, + "after": { + "active_bytes_current": 51523350016, + "active_bytes_peak": 68273233408, + "allocated_bytes_current": 51523350016, + "allocated_bytes_peak": 68273233408, + "device": 1, + "inactive_split_bytes_current": 670568960, + "inactive_split_bytes_peak": 4423937024, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 71470940160, + "reserved_bytes_peak": 71470940160, + "segment_count_current": 702, + "segment_count_peak": 702, + "timestamp_utc": "2026-09-21T07:58:46.453821Z", + "world_size": 8 + }, + "allocated_bytes_current": 51523350016, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68273233408, + "allocated_bytes_peak_delta": 16749880320, + "before": { + "active_bytes_current": 51523350016, + "active_bytes_peak": 51523353088, + "allocated_bytes_current": 51523350016, + "allocated_bytes_peak": 51523353088, + "device": 1, + "inactive_split_bytes_current": 670568960, + "inactive_split_bytes_peak": 2988295680, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 57036242944, + "reserved_bytes_peak": 57036242944, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-09-21T07:58:46.284605Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 670568960, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4423937024, + "inactive_split_bytes_peak_delta": 1435641344, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 71470940160, + "reserved_bytes_current_delta": 14434697216, + "reserved_bytes_peak": 71470940160, + "reserved_bytes_peak_delta": 14434697216, + "segment_count_current": 702, + "segment_count_current_delta": 73, + "segment_count_peak": 702, + "segment_count_peak_delta": 73, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260921T075846454221Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.463934Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260921T075846454221Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260921T075846454221Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fe0166230d87772d345369babff2c1b5ce7f3eaa --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260921T075846454221Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75e137da77edf4b85ea0f855601d9dbefc771e6fcaadfdd5507729f7ebc592a0 +size 570600 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260921T075846579030Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260921T075846579030Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f3fd58d084365c1a1cdea9686ac96953ecc822f3 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260921T075846579030Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51523350016, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68273233408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51523350016, + "active_bytes_peak": 68273233408, + "allocated_bytes_current": 51523350016, + "allocated_bytes_peak": 68273233408, + "device": 1, + "inactive_split_bytes_current": 670568960, + "inactive_split_bytes_peak": 4980468736, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 71470940160, + "reserved_bytes_peak": 71470940160, + "segment_count_current": 702, + "segment_count_peak": 702, + "timestamp_utc": "2026-09-21T07:58:46.578655Z", + "world_size": 8 + }, + "allocated_bytes_current": 51523350016, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68273233408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51523350016, + "active_bytes_peak": 68273233408, + "allocated_bytes_current": 51523350016, + "allocated_bytes_peak": 68273233408, + "device": 1, + "inactive_split_bytes_current": 670568960, + "inactive_split_bytes_peak": 4423937024, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 71470940160, + "reserved_bytes_peak": 71470940160, + "segment_count_current": 702, + "segment_count_peak": 702, + "timestamp_utc": "2026-09-21T07:58:46.465472Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 670568960, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4980468736, + "inactive_split_bytes_peak_delta": 556531712, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 71470940160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71470940160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 702, + "segment_count_current_delta": 0, + "segment_count_peak": 702, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260921T075846579030Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.588669Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260921T075846579030Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260921T075846579030Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a67afc2f8ebd5f126af5236d866a426b7ebbabd4 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260921T075846579030Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed0939b52034fcb7f9953705ed9ca7f6e624c43d293e849d23d405a96753750d +size 571727 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260921T075847870801Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260921T075847870801Z.json new file mode 100644 index 0000000000000000000000000000000000000000..660fd13e3ea75481ad3e4af75b8f806d994da2fc --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260921T075847870801Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51540393472, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 68279731200, + "active_bytes_peak_delta": 6497792, + "after": { + "active_bytes_current": 51540393472, + "active_bytes_peak": 68279731200, + "allocated_bytes_current": 51540393472, + "allocated_bytes_peak": 68279731200, + "device": 1, + "inactive_split_bytes_current": 2838757888, + "inactive_split_bytes_peak": 4980468736, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 72049754112, + "reserved_bytes_peak": 72049754112, + "segment_count_current": 727, + "segment_count_peak": 727, + "timestamp_utc": "2026-09-21T07:58:47.870344Z", + "world_size": 8 + }, + "allocated_bytes_current": 51540393472, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 68279731200, + "allocated_bytes_peak_delta": 6497792, + "before": { + "active_bytes_current": 51523350016, + "active_bytes_peak": 68273233408, + "allocated_bytes_current": 51523350016, + "allocated_bytes_peak": 68273233408, + "device": 1, + "inactive_split_bytes_current": 670568960, + "inactive_split_bytes_peak": 4980468736, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 71470940160, + "reserved_bytes_peak": 71470940160, + "segment_count_current": 702, + "segment_count_peak": 702, + "timestamp_utc": "2026-09-21T07:58:46.590136Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2838757888, + "inactive_split_bytes_current_delta": 2168188928, + "inactive_split_bytes_peak": 4980468736, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 72049754112, + "reserved_bytes_current_delta": 578813952, + "reserved_bytes_peak": 72049754112, + "reserved_bytes_peak_delta": 578813952, + "segment_count_current": 727, + "segment_count_current_delta": 25, + "segment_count_peak": 727, + "segment_count_peak_delta": 25, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260921T075847870801Z.pickle", + "timestamp_utc": "2026-09-21T07:58:47.883430Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260921T075847870801Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260921T075847870801Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..796a65e898ce6efddb674aeebb4237318c1cbcc6 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260921T075847870801Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e77b8c9228b4f159be950290686b27a507ced58c37dd2e443aa3568e60ac19f6 +size 842684 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260921T075847997360Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260921T075847997360Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d44cba798c7fdd77ab54f674c2ef52771a654dac --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260921T075847997360Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51540393472, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68279731200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51540393472, + "active_bytes_peak": 68279731200, + "allocated_bytes_current": 51540393472, + "allocated_bytes_peak": 68279731200, + "device": 1, + "inactive_split_bytes_current": 2838757888, + "inactive_split_bytes_peak": 4980468736, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 72049754112, + "reserved_bytes_peak": 72049754112, + "segment_count_current": 727, + "segment_count_peak": 727, + "timestamp_utc": "2026-09-21T07:58:47.996981Z", + "world_size": 8 + }, + "allocated_bytes_current": 51540393472, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68279731200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51540393472, + "active_bytes_peak": 68279731200, + "allocated_bytes_current": 51540393472, + "allocated_bytes_peak": 68279731200, + "device": 1, + "inactive_split_bytes_current": 2838757888, + "inactive_split_bytes_peak": 4980468736, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 72049754112, + "reserved_bytes_peak": 72049754112, + "segment_count_current": 727, + "segment_count_peak": 727, + "timestamp_utc": "2026-09-21T07:58:47.885113Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2838757888, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4980468736, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 72049754112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 72049754112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 727, + "segment_count_current_delta": 0, + "segment_count_peak": 727, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260921T075847997360Z.pickle", + "timestamp_utc": "2026-09-21T07:58:48.001856Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260921T075847997360Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260921T075847997360Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..220f7e9aa512d9feeea2053d471c03a9dc316418 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260921T075847997360Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e316d92d424fcd9f8f39daeb70a3b1879b5c4bbe4c91a63a2e8ee3931406ba3e +size 247819 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260921T075848115958Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260921T075848115958Z.json new file mode 100644 index 0000000000000000000000000000000000000000..058f51e901a0965c2cfce9efd0c2ac4707769e7a --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260921T075848115958Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51540393472, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68279731200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51540393472, + "active_bytes_peak": 68279731200, + "allocated_bytes_current": 51540393472, + "allocated_bytes_peak": 68279731200, + "device": 1, + "inactive_split_bytes_current": 2838757888, + "inactive_split_bytes_peak": 4980468736, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 72049754112, + "reserved_bytes_peak": 72049754112, + "segment_count_current": 727, + "segment_count_peak": 727, + "timestamp_utc": "2026-09-21T07:58:48.115582Z", + "world_size": 8 + }, + "allocated_bytes_current": 51540393472, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68279731200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51540393472, + "active_bytes_peak": 68279731200, + "allocated_bytes_current": 51540393472, + "allocated_bytes_peak": 68279731200, + "device": 1, + "inactive_split_bytes_current": 2838757888, + "inactive_split_bytes_peak": 4980468736, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 72049754112, + "reserved_bytes_peak": 72049754112, + "segment_count_current": 727, + "segment_count_peak": 727, + "timestamp_utc": "2026-09-21T07:58:48.002712Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2838757888, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4980468736, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 72049754112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 72049754112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 727, + "segment_count_current_delta": 0, + "segment_count_peak": 727, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260921T075848115958Z.pickle", + "timestamp_utc": "2026-09-21T07:58:48.120344Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260921T075848115958Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260921T075848115958Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d638bb1ef1f71b7ac23e15b486fd9be1bf89d23e --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260921T075848115958Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5743100c5e17cefd6960a26c2cd85a12e64a5a0df7a5509c4a0af401b36ae47b +size 247819 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260921T075846268573Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260921T075846268573Z.json new file mode 100644 index 0000000000000000000000000000000000000000..80f759f330d7a660e9af5744c7ac91a6c8b94c42 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260921T075846268573Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51524251136, + "active_bytes_current_delta": 21495818752, + "active_bytes_peak": 51524254208, + "active_bytes_peak_delta": 21437101568, + "after": { + "active_bytes_current": 51524251136, + "active_bytes_peak": 51524254208, + "allocated_bytes_current": 51524251136, + "allocated_bytes_peak": 51524254208, + "device": 2, + "inactive_split_bytes_current": 663376384, + "inactive_split_bytes_peak": 2980646912, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 57036242944, + "reserved_bytes_peak": 57036242944, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-09-21T07:58:46.268040Z", + "world_size": 8 + }, + "allocated_bytes_current": 51524251136, + "allocated_bytes_current_delta": 21495818752, + "allocated_bytes_peak": 51524254208, + "allocated_bytes_peak_delta": 21437101568, + "before": { + "active_bytes_current": 30028432384, + "active_bytes_peak": 30087152640, + "allocated_bytes_current": 30028432384, + "allocated_bytes_peak": 30087152640, + "device": 2, + "inactive_split_bytes_current": 174750720, + "inactive_split_bytes_peak": 761953280, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 35888562176, + "reserved_bytes_peak": 35888562176, + "segment_count_current": 243, + "segment_count_peak": 243, + "timestamp_utc": "2026-09-21T07:58:17.715061Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 663376384, + "inactive_split_bytes_current_delta": 488625664, + "inactive_split_bytes_peak": 2980646912, + "inactive_split_bytes_peak_delta": 2218693632, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 57036242944, + "reserved_bytes_current_delta": 21147680768, + "reserved_bytes_peak": 57036242944, + "reserved_bytes_peak_delta": 21147680768, + "segment_count_current": 629, + "segment_count_current_delta": 386, + "segment_count_peak": 629, + "segment_count_peak_delta": 386, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260921T075846268573Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.283004Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260921T075846268573Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260921T075846268573Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..88f3c2e01d3970daa1aa948a0f2172057e574367 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260921T075846268573Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28e4a4dc646a6d1893dbe750ec7371710bd7b37503cd0db6ee51da3163d2717c +size 714999 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260921T075846454383Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260921T075846454383Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a689831fe1643f2c1b18171b5856f1755bfd0725 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260921T075846454383Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51524251136, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68273265152, + "active_bytes_peak_delta": 16749010944, + "after": { + "active_bytes_current": 51524251136, + "active_bytes_peak": 68273265152, + "allocated_bytes_current": 51524251136, + "allocated_bytes_peak": 68273265152, + "device": 2, + "inactive_split_bytes_current": 663376384, + "inactive_split_bytes_peak": 4418662912, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 701, + "segment_count_peak": 701, + "timestamp_utc": "2026-09-21T07:58:46.453964Z", + "world_size": 8 + }, + "allocated_bytes_current": 51524251136, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68273265152, + "allocated_bytes_peak_delta": 16749010944, + "before": { + "active_bytes_current": 51524251136, + "active_bytes_peak": 51524254208, + "allocated_bytes_current": 51524251136, + "allocated_bytes_peak": 51524254208, + "device": 2, + "inactive_split_bytes_current": 663376384, + "inactive_split_bytes_peak": 2980646912, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 57036242944, + "reserved_bytes_peak": 57036242944, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-09-21T07:58:46.285154Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 663376384, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4418662912, + "inactive_split_bytes_peak_delta": 1438016000, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 71468843008, + "reserved_bytes_current_delta": 14432600064, + "reserved_bytes_peak": 71468843008, + "reserved_bytes_peak_delta": 14432600064, + "segment_count_current": 701, + "segment_count_current_delta": 72, + "segment_count_peak": 701, + "segment_count_peak_delta": 72, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260921T075846454383Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.464529Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260921T075846454383Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260921T075846454383Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d286f3397ec20d9256bfc09c8e930d0feb50986a --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260921T075846454383Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c211660bad3b0af09f75b806b474b10f7b89dc9c8d9c483b0b1566dc159e030 +size 570019 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260921T075846579189Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260921T075846579189Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2450205475430bfd19c063f159a3973e6496c2a6 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260921T075846579189Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51524251136, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68273265152, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51524251136, + "active_bytes_peak": 68273265152, + "allocated_bytes_current": 51524251136, + "allocated_bytes_peak": 68273265152, + "device": 2, + "inactive_split_bytes_current": 663376384, + "inactive_split_bytes_peak": 4975194624, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 701, + "segment_count_peak": 701, + "timestamp_utc": "2026-09-21T07:58:46.578781Z", + "world_size": 8 + }, + "allocated_bytes_current": 51524251136, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68273265152, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51524251136, + "active_bytes_peak": 68273265152, + "allocated_bytes_current": 51524251136, + "allocated_bytes_peak": 68273265152, + "device": 2, + "inactive_split_bytes_current": 663376384, + "inactive_split_bytes_peak": 4418662912, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 701, + "segment_count_peak": 701, + "timestamp_utc": "2026-09-21T07:58:46.466123Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 663376384, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4975194624, + "inactive_split_bytes_peak_delta": 556531712, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 71468843008, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71468843008, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 701, + "segment_count_current_delta": 0, + "segment_count_peak": 701, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260921T075846579189Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.589289Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260921T075846579189Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260921T075846579189Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c200e6e6bb0b3eba63f2e62cd7ca15abb14d4dc5 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260921T075846579189Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1c742a8a2f9c26bb88e0f488f7a1bcfd9037888bdebbff99d49bea8fd83719f +size 571069 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260921T075847870988Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260921T075847870988Z.json new file mode 100644 index 0000000000000000000000000000000000000000..895d8a5e74f0578d50e6d7a01e2a653cda71c2e6 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260921T075847870988Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51541294592, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 68280632320, + "active_bytes_peak_delta": 7367168, + "after": { + "active_bytes_current": 51541294592, + "active_bytes_peak": 68280632320, + "allocated_bytes_current": 51541294592, + "allocated_bytes_peak": 68280632320, + "device": 2, + "inactive_split_bytes_current": 2831565312, + "inactive_split_bytes_peak": 4975194624, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 72043462656, + "reserved_bytes_peak": 72043462656, + "segment_count_current": 724, + "segment_count_peak": 724, + "timestamp_utc": "2026-09-21T07:58:47.870490Z", + "world_size": 8 + }, + "allocated_bytes_current": 51541294592, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 68280632320, + "allocated_bytes_peak_delta": 7367168, + "before": { + "active_bytes_current": 51524251136, + "active_bytes_peak": 68273265152, + "allocated_bytes_current": 51524251136, + "allocated_bytes_peak": 68273265152, + "device": 2, + "inactive_split_bytes_current": 663376384, + "inactive_split_bytes_peak": 4975194624, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 701, + "segment_count_peak": 701, + "timestamp_utc": "2026-09-21T07:58:46.590834Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2831565312, + "inactive_split_bytes_current_delta": 2168188928, + "inactive_split_bytes_peak": 4975194624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 72043462656, + "reserved_bytes_current_delta": 574619648, + "reserved_bytes_peak": 72043462656, + "reserved_bytes_peak_delta": 574619648, + "segment_count_current": 724, + "segment_count_current_delta": 23, + "segment_count_peak": 724, + "segment_count_peak_delta": 23, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260921T075847870988Z.pickle", + "timestamp_utc": "2026-09-21T07:58:47.883607Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260921T075847870988Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260921T075847870988Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4670b60a642153fd38918deffa340365663354d4 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260921T075847870988Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e88e49d393599e3d485b18fe8c99cd5b5a560bc86419997dc69df1e9b8e1d8b +size 841751 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260921T075847997464Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260921T075847997464Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5e98200983834b84bd3553d30bcaf52465e5970d --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260921T075847997464Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51541294592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68280632320, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51541294592, + "active_bytes_peak": 68280632320, + "allocated_bytes_current": 51541294592, + "allocated_bytes_peak": 68280632320, + "device": 2, + "inactive_split_bytes_current": 2831565312, + "inactive_split_bytes_peak": 4975194624, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 72043462656, + "reserved_bytes_peak": 72043462656, + "segment_count_current": 724, + "segment_count_peak": 724, + "timestamp_utc": "2026-09-21T07:58:47.997077Z", + "world_size": 8 + }, + "allocated_bytes_current": 51541294592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68280632320, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51541294592, + "active_bytes_peak": 68280632320, + "allocated_bytes_current": 51541294592, + "allocated_bytes_peak": 68280632320, + "device": 2, + "inactive_split_bytes_current": 2831565312, + "inactive_split_bytes_peak": 4975194624, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 72043462656, + "reserved_bytes_peak": 72043462656, + "segment_count_current": 724, + "segment_count_peak": 724, + "timestamp_utc": "2026-09-21T07:58:47.885384Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2831565312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4975194624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 72043462656, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 72043462656, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 724, + "segment_count_current_delta": 0, + "segment_count_peak": 724, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260921T075847997464Z.pickle", + "timestamp_utc": "2026-09-21T07:58:48.002099Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260921T075847997464Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260921T075847997464Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2b241e2932986e540758676ab96b8bbc25165da3 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260921T075847997464Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbc7ed68eeb0dfa1f90d138735596ee19449fad081223ed6d5e97908861e4141 +size 247079 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260921T075848116117Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260921T075848116117Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7e684051a96c420eb2ea00076c349c4535fdda30 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260921T075848116117Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51541294592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68280632320, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51541294592, + "active_bytes_peak": 68280632320, + "allocated_bytes_current": 51541294592, + "allocated_bytes_peak": 68280632320, + "device": 2, + "inactive_split_bytes_current": 2831565312, + "inactive_split_bytes_peak": 4975194624, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 72043462656, + "reserved_bytes_peak": 72043462656, + "segment_count_current": 724, + "segment_count_peak": 724, + "timestamp_utc": "2026-09-21T07:58:48.115730Z", + "world_size": 8 + }, + "allocated_bytes_current": 51541294592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68280632320, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51541294592, + "active_bytes_peak": 68280632320, + "allocated_bytes_current": 51541294592, + "allocated_bytes_peak": 68280632320, + "device": 2, + "inactive_split_bytes_current": 2831565312, + "inactive_split_bytes_peak": 4975194624, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 72043462656, + "reserved_bytes_peak": 72043462656, + "segment_count_current": 724, + "segment_count_peak": 724, + "timestamp_utc": "2026-09-21T07:58:48.002958Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2831565312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4975194624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 72043462656, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 72043462656, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 724, + "segment_count_current_delta": 0, + "segment_count_peak": 724, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260921T075848116117Z.pickle", + "timestamp_utc": "2026-09-21T07:58:48.120680Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260921T075848116117Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260921T075848116117Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7809d55c5e7a8a52578ffb130dc461558c4d9a9b --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260921T075848116117Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b2dc1555916f97d031ac15ba5c13882ef1bda035aab844a42bf0c7221f069bf +size 247079 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260921T075846268508Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260921T075846268508Z.json new file mode 100644 index 0000000000000000000000000000000000000000..42668b15ee02767f4adeabdaad7ef685acf61d34 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260921T075846268508Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51522939392, + "active_bytes_current_delta": 21495492608, + "active_bytes_peak": 51522942464, + "active_bytes_peak_delta": 21436775424, + "after": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 51522942464, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 51522942464, + "device": 3, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_peak": 2981800960, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 57036242944, + "reserved_bytes_peak": 57036242944, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-09-21T07:58:46.268005Z", + "world_size": 8 + }, + "allocated_bytes_current": 51522939392, + "allocated_bytes_current_delta": 21495492608, + "allocated_bytes_peak": 51522942464, + "allocated_bytes_peak_delta": 21436775424, + "before": { + "active_bytes_current": 30027446784, + "active_bytes_peak": 30086167040, + "allocated_bytes_current": 30027446784, + "allocated_bytes_peak": 30086167040, + "device": 3, + "inactive_split_bytes_current": 175736320, + "inactive_split_bytes_peak": 762938880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 35888562176, + "reserved_bytes_peak": 35888562176, + "segment_count_current": 243, + "segment_count_peak": 243, + "timestamp_utc": "2026-09-21T07:58:17.715232Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_current_delta": 547672064, + "inactive_split_bytes_peak": 2981800960, + "inactive_split_bytes_peak_delta": 2218862080, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 57036242944, + "reserved_bytes_current_delta": 21147680768, + "reserved_bytes_peak": 57036242944, + "reserved_bytes_peak_delta": 21147680768, + "segment_count_current": 629, + "segment_count_current_delta": 386, + "segment_count_peak": 629, + "segment_count_peak_delta": 386, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260921T075846268508Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.282808Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260921T075846268508Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260921T075846268508Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..808ef915bfdf21c48b776bf05c9c35709a4ccd86 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260921T075846268508Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee4324db51cbd12d653caaf3f31630f7b78146191f9b3bb4c808f90245591d70 +size 714281 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260921T075846454411Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260921T075846454411Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c89c1c0b65f209cc64fd5e776d52d049897196fd --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260921T075846454411Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51522939392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68275082240, + "active_bytes_peak_delta": 16752139776, + "after": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 68275082240, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 68275082240, + "device": 3, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_peak": 4463477760, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 699, + "segment_count_peak": 699, + "timestamp_utc": "2026-09-21T07:58:46.453987Z", + "world_size": 8 + }, + "allocated_bytes_current": 51522939392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68275082240, + "allocated_bytes_peak_delta": 16752139776, + "before": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 51522942464, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 51522942464, + "device": 3, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_peak": 2981800960, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 57036242944, + "reserved_bytes_peak": 57036242944, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-09-21T07:58:46.284947Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4463477760, + "inactive_split_bytes_peak_delta": 1481676800, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 71468843008, + "reserved_bytes_current_delta": 14432600064, + "reserved_bytes_peak": 71468843008, + "reserved_bytes_peak_delta": 14432600064, + "segment_count_current": 699, + "segment_count_current_delta": 70, + "segment_count_peak": 699, + "segment_count_peak_delta": 70, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260921T075846454411Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.464508Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260921T075846454411Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260921T075846454411Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6469ddd16dc162492c6cbbdb6a247cbd58b96be5 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260921T075846454411Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b1b0524c0c872cea17bd3c1ee4115840b82c10051b7cc01fbbc6808627e0870 +size 569448 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260921T075846579195Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260921T075846579195Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1aa0616bea2dc0dc2189347ee68bb4bdbd3628fa --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260921T075846579195Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51522939392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68275082240, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 68275082240, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 68275082240, + "device": 3, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_peak": 5020271616, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 699, + "segment_count_peak": 699, + "timestamp_utc": "2026-09-21T07:58:46.578793Z", + "world_size": 8 + }, + "allocated_bytes_current": 51522939392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68275082240, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 68275082240, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 68275082240, + "device": 3, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_peak": 4463477760, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 699, + "segment_count_peak": 699, + "timestamp_utc": "2026-09-21T07:58:46.466145Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5020271616, + "inactive_split_bytes_peak_delta": 556793856, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 71468843008, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71468843008, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 699, + "segment_count_current_delta": 0, + "segment_count_peak": 699, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260921T075846579195Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.589342Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260921T075846579195Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260921T075846579195Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6674f1c339d18e047eb1afd9bd823afe26e04727 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260921T075846579195Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d9f047bcb6f38ba914beb8a8c2644c88e0c9db3a442c55d45a51a21bc33dba0 +size 569556 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260921T075847870964Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260921T075847870964Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ed7ebbd37b404e79f52b77b7df4ca96a4941c28e --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260921T075847870964Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51539982848, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 68279320576, + "active_bytes_peak_delta": 4238336, + "after": { + "active_bytes_current": 51539982848, + "active_bytes_peak": 68279320576, + "allocated_bytes_current": 51539982848, + "allocated_bytes_peak": 68279320576, + "device": 3, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_peak": 5032832512, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 72102182912, + "reserved_bytes_peak": 72102182912, + "segment_count_current": 726, + "segment_count_peak": 726, + "timestamp_utc": "2026-09-21T07:58:47.870481Z", + "world_size": 8 + }, + "allocated_bytes_current": 51539982848, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 68279320576, + "allocated_bytes_peak_delta": 4238336, + "before": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 68275082240, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 68275082240, + "device": 3, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_peak": 5020271616, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 699, + "segment_count_peak": 699, + "timestamp_utc": "2026-09-21T07:58:46.590909Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_current_delta": 2168188928, + "inactive_split_bytes_peak": 5032832512, + "inactive_split_bytes_peak_delta": 12560896, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 72102182912, + "reserved_bytes_current_delta": 633339904, + "reserved_bytes_peak": 72102182912, + "reserved_bytes_peak_delta": 633339904, + "segment_count_current": 726, + "segment_count_current_delta": 27, + "segment_count_peak": 726, + "segment_count_peak_delta": 27, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260921T075847870964Z.pickle", + "timestamp_utc": "2026-09-21T07:58:47.884107Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260921T075847870964Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260921T075847870964Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..51dedd2e36fdce9eb46cdcc80c493e9bf02c5d9c --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260921T075847870964Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c399eeb2d3d4ce9548eafb4672d9476be32744118b1ffbec0ad81497de4af299 +size 842528 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260921T075847997524Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260921T075847997524Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1d99bb14d57d2b0535bd05de98a7d1b473ce2426 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260921T075847997524Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51539982848, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68279320576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51539982848, + "active_bytes_peak": 68279320576, + "allocated_bytes_current": 51539982848, + "allocated_bytes_peak": 68279320576, + "device": 3, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_peak": 5032832512, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 72102182912, + "reserved_bytes_peak": 72102182912, + "segment_count_current": 726, + "segment_count_peak": 726, + "timestamp_utc": "2026-09-21T07:58:47.997104Z", + "world_size": 8 + }, + "allocated_bytes_current": 51539982848, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68279320576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51539982848, + "active_bytes_peak": 68279320576, + "allocated_bytes_current": 51539982848, + "allocated_bytes_peak": 68279320576, + "device": 3, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_peak": 5032832512, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 72102182912, + "reserved_bytes_peak": 72102182912, + "segment_count_current": 726, + "segment_count_peak": 726, + "timestamp_utc": "2026-09-21T07:58:47.885890Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5032832512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 72102182912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 72102182912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 726, + "segment_count_current_delta": 0, + "segment_count_peak": 726, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260921T075847997524Z.pickle", + "timestamp_utc": "2026-09-21T07:58:48.002132Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260921T075847997524Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260921T075847997524Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..608788f0a84958855d69e73cf3642b9831ea2f77 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260921T075847997524Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12e787072af71f3e219b0e3352edcf10be799cac9f678824888426713b22d54c +size 246608 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260921T075848116115Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260921T075848116115Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a5194f9a70e83a62c4f1550ee34376c3886fe49b --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260921T075848116115Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51539982848, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68279320576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51539982848, + "active_bytes_peak": 68279320576, + "allocated_bytes_current": 51539982848, + "allocated_bytes_peak": 68279320576, + "device": 3, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_peak": 5032832512, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 72102182912, + "reserved_bytes_peak": 72102182912, + "segment_count_current": 726, + "segment_count_peak": 726, + "timestamp_utc": "2026-09-21T07:58:48.115723Z", + "world_size": 8 + }, + "allocated_bytes_current": 51539982848, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68279320576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51539982848, + "active_bytes_peak": 68279320576, + "allocated_bytes_current": 51539982848, + "allocated_bytes_peak": 68279320576, + "device": 3, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_peak": 5032832512, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 72102182912, + "reserved_bytes_peak": 72102182912, + "segment_count_current": 726, + "segment_count_peak": 726, + "timestamp_utc": "2026-09-21T07:58:48.002988Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5032832512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 72102182912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 72102182912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 726, + "segment_count_current_delta": 0, + "segment_count_peak": 726, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260921T075848116115Z.pickle", + "timestamp_utc": "2026-09-21T07:58:48.120621Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260921T075848116115Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260921T075848116115Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b185a76efbf90581f155af377cdbf02c31928019 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260921T075848116115Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef2e26a97ea5ef03508cede4315e2efef9183b2f526e3ab9db3e5f11c30eec29 +size 246608 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260921T075846268518Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260921T075846268518Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8b1106df5ef65a363cfe4eb8585d5f55d38f7533 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260921T075846268518Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51522939392, + "active_bytes_current_delta": 21495492608, + "active_bytes_peak": 51522942464, + "active_bytes_peak_delta": 21436775424, + "after": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 51522942464, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 51522942464, + "device": 4, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_peak": 2981800960, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 57036242944, + "reserved_bytes_peak": 57036242944, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-09-21T07:58:46.268023Z", + "world_size": 8 + }, + "allocated_bytes_current": 51522939392, + "allocated_bytes_current_delta": 21495492608, + "allocated_bytes_peak": 51522942464, + "allocated_bytes_peak_delta": 21436775424, + "before": { + "active_bytes_current": 30027446784, + "active_bytes_peak": 30086167040, + "allocated_bytes_current": 30027446784, + "allocated_bytes_peak": 30086167040, + "device": 4, + "inactive_split_bytes_current": 175736320, + "inactive_split_bytes_peak": 762938880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 35888562176, + "reserved_bytes_peak": 35888562176, + "segment_count_current": 243, + "segment_count_peak": 243, + "timestamp_utc": "2026-09-21T07:58:17.715187Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_current_delta": 547672064, + "inactive_split_bytes_peak": 2981800960, + "inactive_split_bytes_peak_delta": 2218862080, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 57036242944, + "reserved_bytes_current_delta": 21147680768, + "reserved_bytes_peak": 57036242944, + "reserved_bytes_peak_delta": 21147680768, + "segment_count_current": 629, + "segment_count_current_delta": 386, + "segment_count_peak": 629, + "segment_count_peak_delta": 386, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260921T075846268518Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.282911Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260921T075846268518Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260921T075846268518Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..25ba68a6aa00474a92b26af895531316cd70e8b8 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260921T075846268518Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d66a8279ec264514c4d61ff7bee7aed29984c8bd94e52bfe2c7a0b63557b7f1 +size 714281 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260921T075846454385Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260921T075846454385Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8303514f2591424be46e721bc57a0a2fc9bb5df7 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260921T075846454385Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51522939392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68275082240, + "active_bytes_peak_delta": 16752139776, + "after": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 68275082240, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 68275082240, + "device": 4, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_peak": 4463477760, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 699, + "segment_count_peak": 699, + "timestamp_utc": "2026-09-21T07:58:46.453969Z", + "world_size": 8 + }, + "allocated_bytes_current": 51522939392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68275082240, + "allocated_bytes_peak_delta": 16752139776, + "before": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 51522942464, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 51522942464, + "device": 4, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_peak": 2981800960, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 57036242944, + "reserved_bytes_peak": 57036242944, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-09-21T07:58:46.285082Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4463477760, + "inactive_split_bytes_peak_delta": 1481676800, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 71468843008, + "reserved_bytes_current_delta": 14432600064, + "reserved_bytes_peak": 71468843008, + "reserved_bytes_peak_delta": 14432600064, + "segment_count_current": 699, + "segment_count_current_delta": 70, + "segment_count_peak": 699, + "segment_count_peak_delta": 70, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260921T075846454385Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.464685Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260921T075846454385Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260921T075846454385Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9ff60ff928d1f91e46d372b8adc55a32a191d2a8 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260921T075846454385Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8ebcb859f2983f7de019d36d20eb0bfb9f21ed5a768a6cf3245c1ac895e24bb +size 569448 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260921T075846579200Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260921T075846579200Z.json new file mode 100644 index 0000000000000000000000000000000000000000..69f5cef92675aa4dc771ba247f8aa589d9a00f54 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260921T075846579200Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51522939392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68275082240, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 68275082240, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 68275082240, + "device": 4, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_peak": 5020271616, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 699, + "segment_count_peak": 699, + "timestamp_utc": "2026-09-21T07:58:46.578780Z", + "world_size": 8 + }, + "allocated_bytes_current": 51522939392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68275082240, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 68275082240, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 68275082240, + "device": 4, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_peak": 4463477760, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 699, + "segment_count_peak": 699, + "timestamp_utc": "2026-09-21T07:58:46.466306Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5020271616, + "inactive_split_bytes_peak_delta": 556793856, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 71468843008, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71468843008, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 699, + "segment_count_current_delta": 0, + "segment_count_peak": 699, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260921T075846579200Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.589717Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260921T075846579200Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260921T075846579200Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3bafb6e08c755c0387fb1f62f566750b149cddd3 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260921T075846579200Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b09ed4846bc734b3a8b58647c1bd85872ae159456355242acecabb6615db0cb +size 569556 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260921T075847870972Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260921T075847870972Z.json new file mode 100644 index 0000000000000000000000000000000000000000..759a5c25823b715aae57cc794c07df4b5e51abe2 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260921T075847870972Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51539982848, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 68279320576, + "active_bytes_peak_delta": 4238336, + "after": { + "active_bytes_current": 51539982848, + "active_bytes_peak": 68279320576, + "allocated_bytes_current": 51539982848, + "allocated_bytes_peak": 68279320576, + "device": 4, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_peak": 5032832512, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 72102182912, + "reserved_bytes_peak": 72102182912, + "segment_count_current": 726, + "segment_count_peak": 726, + "timestamp_utc": "2026-09-21T07:58:47.870477Z", + "world_size": 8 + }, + "allocated_bytes_current": 51539982848, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 68279320576, + "allocated_bytes_peak_delta": 4238336, + "before": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 68275082240, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 68275082240, + "device": 4, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_peak": 5020271616, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 699, + "segment_count_peak": 699, + "timestamp_utc": "2026-09-21T07:58:46.591262Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_current_delta": 2168188928, + "inactive_split_bytes_peak": 5032832512, + "inactive_split_bytes_peak_delta": 12560896, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 72102182912, + "reserved_bytes_current_delta": 633339904, + "reserved_bytes_peak": 72102182912, + "reserved_bytes_peak_delta": 633339904, + "segment_count_current": 726, + "segment_count_current_delta": 27, + "segment_count_peak": 726, + "segment_count_peak_delta": 27, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260921T075847870972Z.pickle", + "timestamp_utc": "2026-09-21T07:58:47.883979Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260921T075847870972Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260921T075847870972Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..222df9f37961e79c088b0834deb204b960e4a804 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260921T075847870972Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5722cba927637c2a93e4feba6c0a007eca02d12a5a6dda673ce669e30bf1d7ae +size 842528 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260921T075847997459Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260921T075847997459Z.json new file mode 100644 index 0000000000000000000000000000000000000000..62fa58a1351f4ec7f1721679f64dd56fafd459e6 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260921T075847997459Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51539982848, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68279320576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51539982848, + "active_bytes_peak": 68279320576, + "allocated_bytes_current": 51539982848, + "allocated_bytes_peak": 68279320576, + "device": 4, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_peak": 5032832512, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 72102182912, + "reserved_bytes_peak": 72102182912, + "segment_count_current": 726, + "segment_count_peak": 726, + "timestamp_utc": "2026-09-21T07:58:47.997070Z", + "world_size": 8 + }, + "allocated_bytes_current": 51539982848, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68279320576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51539982848, + "active_bytes_peak": 68279320576, + "allocated_bytes_current": 51539982848, + "allocated_bytes_peak": 68279320576, + "device": 4, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_peak": 5032832512, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 72102182912, + "reserved_bytes_peak": 72102182912, + "segment_count_current": 726, + "segment_count_peak": 726, + "timestamp_utc": "2026-09-21T07:58:47.885751Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5032832512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 72102182912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 72102182912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 726, + "segment_count_current_delta": 0, + "segment_count_peak": 726, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260921T075847997459Z.pickle", + "timestamp_utc": "2026-09-21T07:58:48.002046Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260921T075847997459Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260921T075847997459Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a6aa53ae0c262392c04091ab876ad512fce43cfb --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260921T075847997459Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4abdbfd8f790d6db7b26700b54447703ada2bc1b2578fd164d42e050d800108 +size 246608 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260921T075848116115Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260921T075848116115Z.json new file mode 100644 index 0000000000000000000000000000000000000000..786794f39706d8e59bdde97aa766dc9d7739ee06 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260921T075848116115Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51539982848, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68279320576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51539982848, + "active_bytes_peak": 68279320576, + "allocated_bytes_current": 51539982848, + "allocated_bytes_peak": 68279320576, + "device": 4, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_peak": 5032832512, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 72102182912, + "reserved_bytes_peak": 72102182912, + "segment_count_current": 726, + "segment_count_peak": 726, + "timestamp_utc": "2026-09-21T07:58:48.115731Z", + "world_size": 8 + }, + "allocated_bytes_current": 51539982848, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68279320576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51539982848, + "active_bytes_peak": 68279320576, + "allocated_bytes_current": 51539982848, + "allocated_bytes_peak": 68279320576, + "device": 4, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_peak": 5032832512, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 72102182912, + "reserved_bytes_peak": 72102182912, + "segment_count_current": 726, + "segment_count_peak": 726, + "timestamp_utc": "2026-09-21T07:58:48.002918Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5032832512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 72102182912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 72102182912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 726, + "segment_count_current_delta": 0, + "segment_count_peak": 726, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260921T075848116115Z.pickle", + "timestamp_utc": "2026-09-21T07:58:48.120734Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260921T075848116115Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260921T075848116115Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..87d3feee0db68270d2db08e502525af62c1d576a --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260921T075848116115Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1786b1b0e46fdab887029071b07a7eb5e742e7890aeffec86c6ddd52d9d9946e +size 246608 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260921T075846268544Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260921T075846268544Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5fa96bd8364a537f2df61df7ce6e81be546607ee --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260921T075846268544Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51524846592, + "active_bytes_current_delta": 21497399808, + "active_bytes_peak": 51524849664, + "active_bytes_peak_delta": 21438682624, + "after": { + "active_bytes_current": 51524846592, + "active_bytes_peak": 51524849664, + "allocated_bytes_current": 51524846592, + "allocated_bytes_peak": 51524849664, + "device": 5, + "inactive_split_bytes_current": 721501184, + "inactive_split_bytes_peak": 2981245952, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 57036242944, + "reserved_bytes_peak": 57036242944, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-09-21T07:58:46.268021Z", + "world_size": 8 + }, + "allocated_bytes_current": 51524846592, + "allocated_bytes_current_delta": 21497399808, + "allocated_bytes_peak": 51524849664, + "allocated_bytes_peak_delta": 21438682624, + "before": { + "active_bytes_current": 30027446784, + "active_bytes_peak": 30086167040, + "allocated_bytes_current": 30027446784, + "allocated_bytes_peak": 30086167040, + "device": 5, + "inactive_split_bytes_current": 175736320, + "inactive_split_bytes_peak": 762938880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 35888562176, + "reserved_bytes_peak": 35888562176, + "segment_count_current": 243, + "segment_count_peak": 243, + "timestamp_utc": "2026-09-21T07:58:17.715243Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 721501184, + "inactive_split_bytes_current_delta": 545764864, + "inactive_split_bytes_peak": 2981245952, + "inactive_split_bytes_peak_delta": 2218307072, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 57036242944, + "reserved_bytes_current_delta": 21147680768, + "reserved_bytes_peak": 57036242944, + "reserved_bytes_peak_delta": 21147680768, + "segment_count_current": 629, + "segment_count_current_delta": 386, + "segment_count_peak": 629, + "segment_count_peak_delta": 386, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260921T075846268544Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.283007Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260921T075846268544Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260921T075846268544Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f460cf681222459af98d90b05a399de7634df31c --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260921T075846268544Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be58b5ad359840901b081ccc230f778b62b408da88804640a96384922b8e9d37 +size 714847 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260921T075846454354Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260921T075846454354Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0e9be8e7f9a8bf4661b595f8b3e9377275d2ef44 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260921T075846454354Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51524846592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68277018112, + "active_bytes_peak_delta": 16752168448, + "after": { + "active_bytes_current": 51524846592, + "active_bytes_peak": 68277018112, + "allocated_bytes_current": 51524846592, + "allocated_bytes_peak": 68277018112, + "device": 5, + "inactive_split_bytes_current": 721501184, + "inactive_split_bytes_peak": 4461345280, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 697, + "segment_count_peak": 697, + "timestamp_utc": "2026-09-21T07:58:46.453929Z", + "world_size": 8 + }, + "allocated_bytes_current": 51524846592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68277018112, + "allocated_bytes_peak_delta": 16752168448, + "before": { + "active_bytes_current": 51524846592, + "active_bytes_peak": 51524849664, + "allocated_bytes_current": 51524846592, + "allocated_bytes_peak": 51524849664, + "device": 5, + "inactive_split_bytes_current": 721501184, + "inactive_split_bytes_peak": 2981245952, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 57036242944, + "reserved_bytes_peak": 57036242944, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-09-21T07:58:46.285219Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 721501184, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4461345280, + "inactive_split_bytes_peak_delta": 1480099328, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 71468843008, + "reserved_bytes_current_delta": 14432600064, + "reserved_bytes_peak": 71468843008, + "reserved_bytes_peak_delta": 14432600064, + "segment_count_current": 697, + "segment_count_current_delta": 68, + "segment_count_peak": 697, + "segment_count_peak_delta": 68, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260921T075846454354Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.464494Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260921T075846454354Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260921T075846454354Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..95ef9ed96695074caa8538cee04febd5271809f0 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260921T075846454354Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28c15f55e307d26afa9a1a66acfc83767e96ba7a31f8dcd49f964782ece98f57 +size 568623 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260921T075846579200Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260921T075846579200Z.json new file mode 100644 index 0000000000000000000000000000000000000000..965676a04791f1a39c93e23cfdfb484e30a43588 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260921T075846579200Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51524846592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68277018112, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51524846592, + "active_bytes_peak": 68277018112, + "allocated_bytes_current": 51524846592, + "allocated_bytes_peak": 68277018112, + "device": 5, + "inactive_split_bytes_current": 721501184, + "inactive_split_bytes_peak": 4930320896, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 697, + "segment_count_peak": 697, + "timestamp_utc": "2026-09-21T07:58:46.578794Z", + "world_size": 8 + }, + "allocated_bytes_current": 51524846592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68277018112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51524846592, + "active_bytes_peak": 68277018112, + "allocated_bytes_current": 51524846592, + "allocated_bytes_peak": 68277018112, + "device": 5, + "inactive_split_bytes_current": 721501184, + "inactive_split_bytes_peak": 4461345280, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 697, + "segment_count_peak": 697, + "timestamp_utc": "2026-09-21T07:58:46.466093Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 721501184, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4930320896, + "inactive_split_bytes_peak_delta": 468975616, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 71468843008, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71468843008, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 697, + "segment_count_current_delta": 0, + "segment_count_peak": 697, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260921T075846579200Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.589411Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260921T075846579200Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260921T075846579200Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d14c5b87af53ec82952f0c497e315f5d5eb7e294 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260921T075846579200Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf1af1f0a53e792783aa8981f28db043631639f30744709e33a62be9a4ca745d +size 569835 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260921T075847870947Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260921T075847870947Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1c69391169032eb6280121ceaf5327683b3515f8 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260921T075847870947Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51541890048, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 68281227776, + "active_bytes_peak_delta": 4209664, + "after": { + "active_bytes_current": 51541890048, + "active_bytes_peak": 68281227776, + "allocated_bytes_current": 51541890048, + "allocated_bytes_peak": 68281227776, + "device": 5, + "inactive_split_bytes_current": 2889690112, + "inactive_split_bytes_peak": 5030925312, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 72102182912, + "reserved_bytes_peak": 72102182912, + "segment_count_current": 726, + "segment_count_peak": 726, + "timestamp_utc": "2026-09-21T07:58:47.870452Z", + "world_size": 8 + }, + "allocated_bytes_current": 51541890048, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 68281227776, + "allocated_bytes_peak_delta": 4209664, + "before": { + "active_bytes_current": 51524846592, + "active_bytes_peak": 68277018112, + "allocated_bytes_current": 51524846592, + "allocated_bytes_peak": 68277018112, + "device": 5, + "inactive_split_bytes_current": 721501184, + "inactive_split_bytes_peak": 4930320896, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 697, + "segment_count_peak": 697, + "timestamp_utc": "2026-09-21T07:58:46.590924Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2889690112, + "inactive_split_bytes_current_delta": 2168188928, + "inactive_split_bytes_peak": 5030925312, + "inactive_split_bytes_peak_delta": 100604416, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 72102182912, + "reserved_bytes_current_delta": 633339904, + "reserved_bytes_peak": 72102182912, + "reserved_bytes_peak_delta": 633339904, + "segment_count_current": 726, + "segment_count_current_delta": 29, + "segment_count_peak": 726, + "segment_count_peak_delta": 29, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260921T075847870947Z.pickle", + "timestamp_utc": "2026-09-21T07:58:47.883879Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260921T075847870947Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260921T075847870947Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0dbe655bbe0884279aac5a249f8d7f9188070168 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260921T075847870947Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85abb8e79cbb8d40cd174260b173d9b64a48e0deaee9c6be8218e562e0fc313e +size 842900 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260921T075847997476Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260921T075847997476Z.json new file mode 100644 index 0000000000000000000000000000000000000000..90e877a0dd286227b03af132b889f0dee161576b --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260921T075847997476Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51541890048, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68281227776, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51541890048, + "active_bytes_peak": 68281227776, + "allocated_bytes_current": 51541890048, + "allocated_bytes_peak": 68281227776, + "device": 5, + "inactive_split_bytes_current": 2889690112, + "inactive_split_bytes_peak": 5030925312, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 72102182912, + "reserved_bytes_peak": 72102182912, + "segment_count_current": 726, + "segment_count_peak": 726, + "timestamp_utc": "2026-09-21T07:58:47.997080Z", + "world_size": 8 + }, + "allocated_bytes_current": 51541890048, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68281227776, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51541890048, + "active_bytes_peak": 68281227776, + "allocated_bytes_current": 51541890048, + "allocated_bytes_peak": 68281227776, + "device": 5, + "inactive_split_bytes_current": 2889690112, + "inactive_split_bytes_peak": 5030925312, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 72102182912, + "reserved_bytes_peak": 72102182912, + "segment_count_current": 726, + "segment_count_peak": 726, + "timestamp_utc": "2026-09-21T07:58:47.885623Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2889690112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5030925312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 72102182912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 72102182912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 726, + "segment_count_current_delta": 0, + "segment_count_peak": 726, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260921T075847997476Z.pickle", + "timestamp_utc": "2026-09-21T07:58:48.002014Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260921T075847997476Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260921T075847997476Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3303158bd223053699afae0ac077b99e5d8fe7e8 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260921T075847997476Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0d1db2068624343cb1a1b3e90d4fa4ebb3da1663f1dab5f2763802fc7591467 +size 247123 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260921T075848116062Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260921T075848116062Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5f6634f177f4efb67caf940e6586b02bcd6f0d5f --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260921T075848116062Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51541890048, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68281227776, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51541890048, + "active_bytes_peak": 68281227776, + "allocated_bytes_current": 51541890048, + "allocated_bytes_peak": 68281227776, + "device": 5, + "inactive_split_bytes_current": 2889690112, + "inactive_split_bytes_peak": 5030925312, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 72102182912, + "reserved_bytes_peak": 72102182912, + "segment_count_current": 726, + "segment_count_peak": 726, + "timestamp_utc": "2026-09-21T07:58:48.115675Z", + "world_size": 8 + }, + "allocated_bytes_current": 51541890048, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68281227776, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51541890048, + "active_bytes_peak": 68281227776, + "allocated_bytes_current": 51541890048, + "allocated_bytes_peak": 68281227776, + "device": 5, + "inactive_split_bytes_current": 2889690112, + "inactive_split_bytes_peak": 5030925312, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 72102182912, + "reserved_bytes_peak": 72102182912, + "segment_count_current": 726, + "segment_count_peak": 726, + "timestamp_utc": "2026-09-21T07:58:48.002863Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2889690112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5030925312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 72102182912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 72102182912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 726, + "segment_count_current_delta": 0, + "segment_count_peak": 726, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260921T075848116062Z.pickle", + "timestamp_utc": "2026-09-21T07:58:48.120537Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260921T075848116062Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260921T075848116062Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1ea178990756b0d3fa5d4cfd5a5ac0bc3dfa85a0 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260921T075848116062Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:539eb79c5e387e07ff9149425ad10122a9098a37bc0f3dc2fa293089cc24acb4 +size 247123 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260921T075846268470Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260921T075846268470Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8369e2a3345c9f2b6b5bcc0741a3f89b1bfd3b1e --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260921T075846268470Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51523586048, + "active_bytes_current_delta": 21496139264, + "active_bytes_peak": 51523589120, + "active_bytes_peak_delta": 21437422080, + "after": { + "active_bytes_current": 51523586048, + "active_bytes_peak": 51523589120, + "allocated_bytes_current": 51523586048, + "allocated_bytes_peak": 51523589120, + "device": 6, + "inactive_split_bytes_current": 752121856, + "inactive_split_bytes_peak": 2981725184, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 57036242944, + "reserved_bytes_peak": 57036242944, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-09-21T07:58:46.267899Z", + "world_size": 8 + }, + "allocated_bytes_current": 51523586048, + "allocated_bytes_current_delta": 21496139264, + "allocated_bytes_peak": 51523589120, + "allocated_bytes_peak_delta": 21437422080, + "before": { + "active_bytes_current": 30027446784, + "active_bytes_peak": 30086167040, + "allocated_bytes_current": 30027446784, + "allocated_bytes_peak": 30086167040, + "device": 6, + "inactive_split_bytes_current": 175736320, + "inactive_split_bytes_peak": 762938880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 35888562176, + "reserved_bytes_peak": 35888562176, + "segment_count_current": 243, + "segment_count_peak": 243, + "timestamp_utc": "2026-09-21T07:58:17.715155Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 752121856, + "inactive_split_bytes_current_delta": 576385536, + "inactive_split_bytes_peak": 2981725184, + "inactive_split_bytes_peak_delta": 2218786304, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 57036242944, + "reserved_bytes_current_delta": 21147680768, + "reserved_bytes_peak": 57036242944, + "reserved_bytes_peak_delta": 21147680768, + "segment_count_current": 629, + "segment_count_current_delta": 386, + "segment_count_peak": 629, + "segment_count_peak_delta": 386, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260921T075846268470Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.282588Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260921T075846268470Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260921T075846268470Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e2ee36a5c394caf39aba0743792e8c990aa7505d --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260921T075846268470Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af9076eae53d010c81a3159a2c65c9f6ef309162fcab8927729a8c93d9f1a03b +size 714929 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260921T075846454249Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260921T075846454249Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8f372fddf874293b8074e6c9c98425c2bf736409 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260921T075846454249Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51523586048, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68274923520, + "active_bytes_peak_delta": 16751334400, + "after": { + "active_bytes_current": 51523586048, + "active_bytes_peak": 68274923520, + "allocated_bytes_current": 51523586048, + "allocated_bytes_peak": 68274923520, + "device": 6, + "inactive_split_bytes_current": 752121856, + "inactive_split_bytes_peak": 4462530048, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 71498203136, + "reserved_bytes_peak": 71498203136, + "segment_count_current": 698, + "segment_count_peak": 698, + "timestamp_utc": "2026-09-21T07:58:46.453842Z", + "world_size": 8 + }, + "allocated_bytes_current": 51523586048, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68274923520, + "allocated_bytes_peak_delta": 16751334400, + "before": { + "active_bytes_current": 51523586048, + "active_bytes_peak": 51523589120, + "allocated_bytes_current": 51523586048, + "allocated_bytes_peak": 51523589120, + "device": 6, + "inactive_split_bytes_current": 752121856, + "inactive_split_bytes_peak": 2981725184, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 57036242944, + "reserved_bytes_peak": 57036242944, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-09-21T07:58:46.284703Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 752121856, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4462530048, + "inactive_split_bytes_peak_delta": 1480804864, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 71498203136, + "reserved_bytes_current_delta": 14461960192, + "reserved_bytes_peak": 71498203136, + "reserved_bytes_peak_delta": 14461960192, + "segment_count_current": 698, + "segment_count_current_delta": 69, + "segment_count_peak": 698, + "segment_count_peak_delta": 69, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260921T075846454249Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.464081Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260921T075846454249Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260921T075846454249Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf0e4139e0214e42a6bbe27a6bc78167e104db04 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260921T075846454249Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dc8e8403e98e7387e5fa2b29cb2d36bbbf9742305949028a593a677a8c72eaf +size 568776 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260921T075846579053Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260921T075846579053Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f860e5390b4f35e4f373e3171d8a647017f386b4 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260921T075846579053Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51523586048, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68274923520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51523586048, + "active_bytes_peak": 68274923520, + "allocated_bytes_current": 51523586048, + "allocated_bytes_peak": 68274923520, + "device": 6, + "inactive_split_bytes_current": 752121856, + "inactive_split_bytes_peak": 4960880128, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 71498203136, + "reserved_bytes_peak": 71498203136, + "segment_count_current": 698, + "segment_count_peak": 698, + "timestamp_utc": "2026-09-21T07:58:46.578657Z", + "world_size": 8 + }, + "allocated_bytes_current": 51523586048, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68274923520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51523586048, + "active_bytes_peak": 68274923520, + "allocated_bytes_current": 51523586048, + "allocated_bytes_peak": 68274923520, + "device": 6, + "inactive_split_bytes_current": 752121856, + "inactive_split_bytes_peak": 4462530048, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 71498203136, + "reserved_bytes_peak": 71498203136, + "segment_count_current": 698, + "segment_count_peak": 698, + "timestamp_utc": "2026-09-21T07:58:46.465630Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 752121856, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4960880128, + "inactive_split_bytes_peak_delta": 498350080, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 71498203136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71498203136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 698, + "segment_count_current_delta": 0, + "segment_count_peak": 698, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260921T075846579053Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.589030Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260921T075846579053Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260921T075846579053Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fbb1272a2edf7940b0f941ecdd44ad3cf95dd7c1 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260921T075846579053Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8510fa14b58f0b0efecf7939b568f73e1f26078cd4b95ea0155add46106ebc4d +size 570032 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260921T075847870841Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260921T075847870841Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d2ebd29d653457b92a0a2ae0dded383cd68e70d9 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260921T075847870841Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51540629504, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 68279967232, + "active_bytes_peak_delta": 5043712, + "after": { + "active_bytes_current": 51540629504, + "active_bytes_peak": 68279967232, + "allocated_bytes_current": 51540629504, + "allocated_bytes_peak": 68279967232, + "device": 6, + "inactive_split_bytes_current": 2920310784, + "inactive_split_bytes_peak": 5061545984, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 72131543040, + "reserved_bytes_peak": 72131543040, + "segment_count_current": 727, + "segment_count_peak": 727, + "timestamp_utc": "2026-09-21T07:58:47.870374Z", + "world_size": 8 + }, + "allocated_bytes_current": 51540629504, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 68279967232, + "allocated_bytes_peak_delta": 5043712, + "before": { + "active_bytes_current": 51523586048, + "active_bytes_peak": 68274923520, + "allocated_bytes_current": 51523586048, + "allocated_bytes_peak": 68274923520, + "device": 6, + "inactive_split_bytes_current": 752121856, + "inactive_split_bytes_peak": 4960880128, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 71498203136, + "reserved_bytes_peak": 71498203136, + "segment_count_current": 698, + "segment_count_peak": 698, + "timestamp_utc": "2026-09-21T07:58:46.590511Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2920310784, + "inactive_split_bytes_current_delta": 2168188928, + "inactive_split_bytes_peak": 5061545984, + "inactive_split_bytes_peak_delta": 100665856, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 72131543040, + "reserved_bytes_current_delta": 633339904, + "reserved_bytes_peak": 72131543040, + "reserved_bytes_peak_delta": 633339904, + "segment_count_current": 727, + "segment_count_current_delta": 29, + "segment_count_peak": 727, + "segment_count_peak_delta": 29, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260921T075847870841Z.pickle", + "timestamp_utc": "2026-09-21T07:58:47.883763Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260921T075847870841Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260921T075847870841Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e3214f14644e11c70ed921c4ed4cc75046743312 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260921T075847870841Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cd72f1dd0bc422ae552e428d1a43157ae87dca1c51a92b32253ce91d01aee2b +size 843121 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260921T075847997384Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260921T075847997384Z.json new file mode 100644 index 0000000000000000000000000000000000000000..06c117099d3e46c08c6b1e13ea0db42580897c1a --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260921T075847997384Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51540629504, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68279967232, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51540629504, + "active_bytes_peak": 68279967232, + "allocated_bytes_current": 51540629504, + "allocated_bytes_peak": 68279967232, + "device": 6, + "inactive_split_bytes_current": 2920310784, + "inactive_split_bytes_peak": 5061545984, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 72131543040, + "reserved_bytes_peak": 72131543040, + "segment_count_current": 727, + "segment_count_peak": 727, + "timestamp_utc": "2026-09-21T07:58:47.996980Z", + "world_size": 8 + }, + "allocated_bytes_current": 51540629504, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68279967232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51540629504, + "active_bytes_peak": 68279967232, + "allocated_bytes_current": 51540629504, + "allocated_bytes_peak": 68279967232, + "device": 6, + "inactive_split_bytes_current": 2920310784, + "inactive_split_bytes_peak": 5061545984, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 72131543040, + "reserved_bytes_peak": 72131543040, + "segment_count_current": 727, + "segment_count_peak": 727, + "timestamp_utc": "2026-09-21T07:58:47.885487Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2920310784, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5061545984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 72131543040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 72131543040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 727, + "segment_count_current_delta": 0, + "segment_count_peak": 727, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260921T075847997384Z.pickle", + "timestamp_utc": "2026-09-21T07:58:48.001901Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260921T075847997384Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260921T075847997384Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dcce48d25e0ec787a73b769106518b6a0242b881 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260921T075847997384Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63de666e5339bbb7359be2bc1ebee33f189c4f50537b4652b5a97fa8381a90da +size 247309 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260921T075848115994Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260921T075848115994Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6741f6758f7e93851f4a8c74bc84841650d74478 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260921T075848115994Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51540629504, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68279967232, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51540629504, + "active_bytes_peak": 68279967232, + "allocated_bytes_current": 51540629504, + "allocated_bytes_peak": 68279967232, + "device": 6, + "inactive_split_bytes_current": 2920310784, + "inactive_split_bytes_peak": 5061545984, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 72131543040, + "reserved_bytes_peak": 72131543040, + "segment_count_current": 727, + "segment_count_peak": 727, + "timestamp_utc": "2026-09-21T07:58:48.115606Z", + "world_size": 8 + }, + "allocated_bytes_current": 51540629504, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68279967232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51540629504, + "active_bytes_peak": 68279967232, + "allocated_bytes_current": 51540629504, + "allocated_bytes_peak": 68279967232, + "device": 6, + "inactive_split_bytes_current": 2920310784, + "inactive_split_bytes_peak": 5061545984, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 72131543040, + "reserved_bytes_peak": 72131543040, + "segment_count_current": 727, + "segment_count_peak": 727, + "timestamp_utc": "2026-09-21T07:58:48.002769Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2920310784, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5061545984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 72131543040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 72131543040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 727, + "segment_count_current_delta": 0, + "segment_count_peak": 727, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260921T075848115994Z.pickle", + "timestamp_utc": "2026-09-21T07:58:48.120677Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260921T075848115994Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260921T075848115994Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6ced74a780583d3091c88ae40ad52a90ed2f2a21 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260921T075848115994Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a52a39ccb3a3db9831214b1bc9064696e66105dec755a034754e5cb3a3e30ad5 +size 247309 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260921T075846268483Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260921T075846268483Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5783f6378ef33825edb1f8fe0cb3310a613dc9d6 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260921T075846268483Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51522939392, + "active_bytes_current_delta": 21495492608, + "active_bytes_peak": 51522942464, + "active_bytes_peak_delta": 21436775424, + "after": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 51522942464, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 51522942464, + "device": 7, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_peak": 2981800960, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 57036242944, + "reserved_bytes_peak": 57036242944, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-09-21T07:58:46.267924Z", + "world_size": 8 + }, + "allocated_bytes_current": 51522939392, + "allocated_bytes_current_delta": 21495492608, + "allocated_bytes_peak": 51522942464, + "allocated_bytes_peak_delta": 21436775424, + "before": { + "active_bytes_current": 30027446784, + "active_bytes_peak": 30086167040, + "allocated_bytes_current": 30027446784, + "allocated_bytes_peak": 30086167040, + "device": 7, + "inactive_split_bytes_current": 175736320, + "inactive_split_bytes_peak": 762938880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 35888562176, + "reserved_bytes_peak": 35888562176, + "segment_count_current": 243, + "segment_count_peak": 243, + "timestamp_utc": "2026-09-21T07:58:17.715021Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_current_delta": 547672064, + "inactive_split_bytes_peak": 2981800960, + "inactive_split_bytes_peak_delta": 2218862080, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 57036242944, + "reserved_bytes_current_delta": 21147680768, + "reserved_bytes_peak": 57036242944, + "reserved_bytes_peak_delta": 21147680768, + "segment_count_current": 629, + "segment_count_current_delta": 386, + "segment_count_peak": 629, + "segment_count_peak_delta": 386, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260921T075846268483Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.282661Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260921T075846268483Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260921T075846268483Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eb00e7a31608a6b640a42e3ead9f5387644aa5be --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260921T075846268483Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76f9d17be2e49b543d1728f965bf87f92df18410071111384062bf0d79a61923 +size 714281 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260921T075846454295Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260921T075846454295Z.json new file mode 100644 index 0000000000000000000000000000000000000000..019bbe410a266d2d3aa3a611c22e0a50c2e0862c --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260921T075846454295Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51522939392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68275082240, + "active_bytes_peak_delta": 16752139776, + "after": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 68275082240, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 68275082240, + "device": 7, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_peak": 4463463424, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 699, + "segment_count_peak": 699, + "timestamp_utc": "2026-09-21T07:58:46.453896Z", + "world_size": 8 + }, + "allocated_bytes_current": 51522939392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68275082240, + "allocated_bytes_peak_delta": 16752139776, + "before": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 51522942464, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 51522942464, + "device": 7, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_peak": 2981800960, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 57036242944, + "reserved_bytes_peak": 57036242944, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-09-21T07:58:46.284788Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4463463424, + "inactive_split_bytes_peak_delta": 1481662464, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 71468843008, + "reserved_bytes_current_delta": 14432600064, + "reserved_bytes_peak": 71468843008, + "reserved_bytes_peak_delta": 14432600064, + "segment_count_current": 699, + "segment_count_current_delta": 70, + "segment_count_peak": 699, + "segment_count_peak_delta": 70, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260921T075846454295Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.463949Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260921T075846454295Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260921T075846454295Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0e5bf0515c62e4d81f00fb1efb593f1719ff477b --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260921T075846454295Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51a632d5b6d32d76fdf18548f89fb37a8cc12d13fb8f7cb1dfd570ea3fdf9f56 +size 569448 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260921T075846579068Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260921T075846579068Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b02a8d425c5066bac453c4b2f5b94bcb9ca695ca --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260921T075846579068Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51522939392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68275082240, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 68275082240, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 68275082240, + "device": 7, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_peak": 5020271616, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 699, + "segment_count_peak": 699, + "timestamp_utc": "2026-09-21T07:58:46.578692Z", + "world_size": 8 + }, + "allocated_bytes_current": 51522939392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68275082240, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 68275082240, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 68275082240, + "device": 7, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_peak": 4463463424, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 699, + "segment_count_peak": 699, + "timestamp_utc": "2026-09-21T07:58:46.465508Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5020271616, + "inactive_split_bytes_peak_delta": 556808192, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 71468843008, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71468843008, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 699, + "segment_count_current_delta": 0, + "segment_count_peak": 699, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260921T075846579068Z.pickle", + "timestamp_utc": "2026-09-21T07:58:46.589052Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260921T075846579068Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260921T075846579068Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..72228c312c96a2c4889a04500184ba980c94ba5a --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260921T075846579068Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:036c30a15b58600f447cb7f274ddfede198845794b3025ee8038bb6ddb65714f +size 569556 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260921T075847870841Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260921T075847870841Z.json new file mode 100644 index 0000000000000000000000000000000000000000..80d7e1292f55fdebcb2239e67f4e570d8f72ba1e --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260921T075847870841Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51539982848, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 68279320576, + "active_bytes_peak_delta": 4238336, + "after": { + "active_bytes_current": 51539982848, + "active_bytes_peak": 68279320576, + "allocated_bytes_current": 51539982848, + "allocated_bytes_peak": 68279320576, + "device": 7, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_peak": 5032832512, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 72102182912, + "reserved_bytes_peak": 72102182912, + "segment_count_current": 726, + "segment_count_peak": 726, + "timestamp_utc": "2026-09-21T07:58:47.870380Z", + "world_size": 8 + }, + "allocated_bytes_current": 51539982848, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 68279320576, + "allocated_bytes_peak_delta": 4238336, + "before": { + "active_bytes_current": 51522939392, + "active_bytes_peak": 68275082240, + "allocated_bytes_current": 51522939392, + "allocated_bytes_peak": 68275082240, + "device": 7, + "inactive_split_bytes_current": 723408384, + "inactive_split_bytes_peak": 5020271616, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 71468843008, + "reserved_bytes_peak": 71468843008, + "segment_count_current": 699, + "segment_count_peak": 699, + "timestamp_utc": "2026-09-21T07:58:46.590530Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_current_delta": 2168188928, + "inactive_split_bytes_peak": 5032832512, + "inactive_split_bytes_peak_delta": 12560896, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 72102182912, + "reserved_bytes_current_delta": 633339904, + "reserved_bytes_peak": 72102182912, + "reserved_bytes_peak_delta": 633339904, + "segment_count_current": 726, + "segment_count_current_delta": 27, + "segment_count_peak": 726, + "segment_count_peak_delta": 27, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260921T075847870841Z.pickle", + "timestamp_utc": "2026-09-21T07:58:47.883752Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260921T075847870841Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260921T075847870841Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..72b0eeefa054d196522db2e2d81c6bee9e724e2f --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260921T075847870841Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:775c0d5d9aecf5c2d025ac15d993f7351096220b2edcd2848b4ca37b0e6cf41e +size 842528 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260921T075847997427Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260921T075847997427Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7568ab47be2834c516c107e548c51ebabe3aff76 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260921T075847997427Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51539982848, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68279320576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51539982848, + "active_bytes_peak": 68279320576, + "allocated_bytes_current": 51539982848, + "allocated_bytes_peak": 68279320576, + "device": 7, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_peak": 5032832512, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 72102182912, + "reserved_bytes_peak": 72102182912, + "segment_count_current": 726, + "segment_count_peak": 726, + "timestamp_utc": "2026-09-21T07:58:47.997037Z", + "world_size": 8 + }, + "allocated_bytes_current": 51539982848, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68279320576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51539982848, + "active_bytes_peak": 68279320576, + "allocated_bytes_current": 51539982848, + "allocated_bytes_peak": 68279320576, + "device": 7, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_peak": 5032832512, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 72102182912, + "reserved_bytes_peak": 72102182912, + "segment_count_current": 726, + "segment_count_peak": 726, + "timestamp_utc": "2026-09-21T07:58:47.885569Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5032832512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 72102182912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 72102182912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 726, + "segment_count_current_delta": 0, + "segment_count_peak": 726, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260921T075847997427Z.pickle", + "timestamp_utc": "2026-09-21T07:58:48.001929Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260921T075847997427Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260921T075847997427Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..903d96d2d9d58a1d97f3092c7e3c4e79b93e22d4 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260921T075847997427Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43178eb76821516de22eaf8201273ba10932e4cbe2f9253f2115be1ba74839aa +size 246608 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260921T075848116016Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260921T075848116016Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fb6ca1201836f7f3fe11c3db9702a484fcaaff58 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260921T075848116016Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51539982848, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68279320576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51539982848, + "active_bytes_peak": 68279320576, + "allocated_bytes_current": 51539982848, + "allocated_bytes_peak": 68279320576, + "device": 7, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_peak": 5032832512, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 72102182912, + "reserved_bytes_peak": 72102182912, + "segment_count_current": 726, + "segment_count_peak": 726, + "timestamp_utc": "2026-09-21T07:58:48.115640Z", + "world_size": 8 + }, + "allocated_bytes_current": 51539982848, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68279320576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51539982848, + "active_bytes_peak": 68279320576, + "allocated_bytes_current": 51539982848, + "allocated_bytes_peak": 68279320576, + "device": 7, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_peak": 5032832512, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 72102182912, + "reserved_bytes_peak": 72102182912, + "segment_count_current": 726, + "segment_count_peak": 726, + "timestamp_utc": "2026-09-21T07:58:48.002791Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2891597312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5032832512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 72102182912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 72102182912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 726, + "segment_count_current_delta": 0, + "segment_count_peak": 726, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260921T075848116016Z.pickle", + "timestamp_utc": "2026-09-21T07:58:48.120519Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260921T075848116016Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260921T075848116016Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6a95a25cf557461dde001070c6c3ddec6065758b --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260921T075848116016Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bb2626dd146d303fc58bdc444357d94255e5f062455a549b53b610089ed65dc +size 246608 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/run.env b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..8716ee431cec349c865f10c0fe67a4708f25dfd2 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/run.env @@ -0,0 +1,102 @@ +backend=ultraep +config=KIMI_K2_L2 +family=tierB +source_model=Kimi-K2 +standard_num_layers=61 +base_rung= +repo=/workspace/Megatron-UltraEP +branch=ultraep-better-memory-dump-support +commit=c3a8a8a7 +num_layers=2 +hidden=7168 +ffn_hidden=18432 +moe_ffn_hidden=2048 +num_experts=384 +topk=8 +experts_per_rank=48 +expert_params=44040192 +heads=64 +kv_groups=64 +kv_channels=64 +qk_layernorm=1 +shared_ffn=2048 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=1 +q_lora_rank=1536 +kv_lora_rank=512 +qk_head_dim=128 +qk_pos_emb_head_dim=64 +v_head_dim=128 +dense_first_layers=1 +moe_layers=1 +rotary_base=50000 +rotary_percent=1.0 +rotary_scaling=32 +mscale=1.0 +mscale_all_dim=1.0 +qkv_bias=0 +router_balancing=seq_aux_loss +aux_loss_coeff=0 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +router_groups=1 +router_group_topk=1 +router_score_function=sigmoid +router_expert_bias=1 +router_bias_update_rate=1e-3 +router_topk_scaling_factor=2.827 +shared_expert_gate=0 +apply_layernorm_1p=0 +attention_output_gate=0 +linear_attention_type= +linear_attention_freq= +linear_conv_kernel_dim= +linear_key_head_dim= +linear_value_head_dim= +linear_num_key_heads= +linear_num_value_heads= +mtp_num_layers=0 +mtp_loss_scaling_factor=0.1 +no_weight_decay_cond_type= +max_position_embeddings=4096 +init_method_std=0.02 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=682 +slot_policy=matched +slots_per_rank=48 +capacity_mode=static +capacity=1.17578125 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=6 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=85 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/train.log b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..6be5b4960a09a646d20bec8425fd087db094a9f1 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/train.log @@ -0,0 +1,1070 @@ +[07:57:49] WARN: ultraep-better-memory-dump-support lacks --moe-router-fusion; KIMI_K2_L2 runs unfused (timing only, routing identical) +============================================================== + KIMI_K2_L2 / ultraep (slots/rank=48 policy=matched) + E=384 K=8 H'=2048 m=48 + L=2 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=1.17578125 cg=1 recompute=none + memory_dump=iteration snapshots=/workspace/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ c3a8a8a7 +============================================================== +W0921 07:57:51.033000 556752 torch/distributed/run.py:766] +W0921 07:57:51.033000 556752 torch/distributed/run.py:766] ***************************************** +W0921 07:57:51.033000 556752 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0921 07:57:51.033000 556752 torch/distributed/run.py:766] ***************************************** +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 2 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 18432 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 7168 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 4096 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_capacity_monitor ............................ True + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 2048 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1] + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 48 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.17578125 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... True + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... 1 + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... 1 + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... sigmoid + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. 2.827 + moe_shared_expert_intermediate_size ............. 2048 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 1.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 64 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 384 + num_layers ...................................... 2 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... 1536 + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 50000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 32.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6[Gloo] Rank [Gloo] Rank is connected to 773 is connected to peer ranks. is connected to 7[Gloo] Rank Expected number of connected peer ranks is : 7 peer ranks. [Gloo] Rank 7 peer ranks. +4Expected number of connected peer ranks is : Expected number of connected peer ranks is : 5 is connected to 77 + is connected to 7 +7 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 0 is connected to 00 is connected to peer ranks. 0Expected number of connected peer ranks is : peer ranks. 0Expected number of connected peer ranks is : +0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank2]:[W921 07:58:08.105781790 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W921 07:58:08.105781862 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W921 07:58:08.105832192 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W921 07:58:08.105887320 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W921 07:58:08.108162187 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W921 07:58:08.109627717 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W921 07:58:08.120876589 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.037 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W921 07:58:08.331769001 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.772 seconds +time to initialize megatron (seconds): 7.361 +[after megatron is initialized] datetime: 2026-09-21 07:58:14 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 5594455040 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 9 +Params for bucket 1 (360579072 elements, 360579072 padded size): + module.output_layer.weight +Params for bucket 2 (44047360 elements, 44047360 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight +Params for bucket 3 (45163520 elements, 45163520 padded size): + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_q_up_proj.weight + module.decoder.layers.1.self_attention.linear_q_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_q_down_proj.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight +Params for bucket 4 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight +Params for bucket 5 (132127744 elements, 132127744 padded size): + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc2.weight +Params for bucket 6 (264241152 elements, 264241152 padded size): + module.decoder.layers.0.mlp.linear_fc1.weight +Params for bucket 7 (42411008 elements, 42411008 padded size): + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_q_up_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_q_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_q_down_proj.weight +Params for bucket 8 (58720256 elements, 58720256 padded size): + module.decoder.layers.0.self_attention.linear_proj.weight +Params for bucket 9 (360586240 elements, 360586240 padded size): + module.decoder.layers.0.input_layernorm.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight47 + module.decoder.layers.1.mlp.experts.linear_fc2.weight45 + module.decoder.layers.1.mlp.experts.linear_fc2.weight46 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight44 + module.decoder.layers.1.mlp.experts.linear_fc2.weight42 + module.decoder.layers.1.mlp.experts.linear_fc2.weight43 +Params for bucket 3 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight41 + module.decoder.layers.1.mlp.experts.linear_fc2.weight39 + module.decoder.layers.1.mlp.experts.linear_fc2.weight40 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight38 + module.decoder.layers.1.mlp.experts.linear_fc2.weight37 + module.decoder.layers.1.mlp.experts.linear_fc2.weight36 +Params for bucket 5 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight35 + module.decoder.layers.1.mlp.experts.linear_fc2.weight34 + module.decoder.layers.1.mlp.experts.linear_fc2.weight33 +Params for bucket 6 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight32 + module.decoder.layers.1.mlp.experts.linear_fc2.weight31 + module.decoder.layers.1.mlp.experts.linear_fc2.weight30 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight29 + module.decoder.layers.1.mlp.experts.linear_fc2.weight28 + module.decoder.layers.1.mlp.experts.linear_fc2.weight27 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight26 + module.decoder.layers.1.mlp.experts.linear_fc2.weight25 + module.decoder.layers.1.mlp.experts.linear_fc2.weight24 +Params for bucket 9 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight23 + module.decoder.layers.1.mlp.experts.linear_fc2.weight21 + module.decoder.layers.1.mlp.experts.linear_fc2.weight22 +Params for bucket 10 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight20 + module.decoder.layers.1.mlp.experts.linear_fc2.weight18 + module.decoder.layers.1.mlp.experts.linear_fc2.weight19 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight17 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight16 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 +Params for bucket 17 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight47 + module.decoder.layers.1.mlp.experts.linear_fc1.weight46 +Params for bucket 18 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight45 + module.decoder.layers.1.mlp.experts.linear_fc1.weight44 +Params for bucket 19 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight43 + module.decoder.layers.1.mlp.experts.linear_fc1.weight42 +Params for bucket 20 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight41 + module.decoder.layers.1.mlp.experts.linear_fc1.weight40 +Params for bucket 21 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight39 + module.decoder.layers.1.mlp.experts.linear_fc1.weight38 +Params for bucket 22 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight37 + module.decoder.layers.1.mlp.experts.linear_fc1.weight36 +Params for bucket 23 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight35 + module.decoder.layers.1.mlp.experts.linear_fc1.weight34 +Params for bucket 24 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight33 + module.decoder.layers.1.mlp.experts.linear_fc1.weight32 +Params for bucket 25 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight31 + module.decoder.layers.1.mlp.experts.linear_fc1.weight30 +Params for bucket 26 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight29 + module.decoder.layers.1.mlp.experts.linear_fc1.weight28 +Params for bucket 27 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight27 + module.decoder.layers.1.mlp.experts.linear_fc1.weight26 +Params for bucket 28 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight25 + module.decoder.layers.1.mlp.experts.linear_fc1.weight24 +Params for bucket 29 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight23 + module.decoder.layers.1.mlp.experts.linear_fc1.weight22 +Params for bucket 30 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight21 + module.decoder.layers.1.mlp.experts.linear_fc1.weight20 +Params for bucket 31 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight19 + module.decoder.layers.1.mlp.experts.linear_fc1.weight18 +Params for bucket 32 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight17 + module.decoder.layers.1.mlp.experts.linear_fc1.weight16 +Params for bucket 33 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 +Params for bucket 34 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 +Params for bucket 35 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 +Params for bucket 36 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 +Params for bucket 37 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 +Params for bucket 38 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 39 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 +Params for bucket 40 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-21 07:58:17 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 2b111c634810f9290a8a96b2225cf512-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-09-21 07:58:17 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3385.68, 3390.32) + train/valid/test-data-iterators-setup ..........: (72.11, 93.26) +[before the start of training step] datetime: 2026-09-21 07:58:17 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-09-21 07:58:46] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 28551.8 | throughput per GPU (TFLOP/s/GPU): 1.2 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.227924E+01 | loss scale: 1.0 | grad norm: 12.617 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 | +Number of parameters in transformer block in billions: 17.55 +Number of parameters in embedding layers in billions: 0.72 +Total number of parameters in billions: 18.28 +Number of parameters in most loaded shard in billions: 18.2753 +Theoretical memory footprints: weight and optimizer=130715.23 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 49136.10400390625 | max allocated: 49136.1044921875 | reserved: 54394.0 | max reserved: 54394.0 +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-09-21 07:58:46] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 186.6 | throughput per GPU (TFLOP/s/GPU): 189.5 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.227088E+01 | loss scale: 1.0 | grad norm: 12.534 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 | + [2026-09-21 07:58:46] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 124.8 | throughput per GPU (TFLOP/s/GPU): 283.5 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.088328E+01 | loss scale: 1.0 | grad norm: 11.023 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36224 | min safe capacity: 1.1055 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-09-21 07:58:47] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 1291.6 | throughput per GPU (TFLOP/s/GPU): 27.4 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.448181E+01 | loss scale: 1.0 | grad norm: 72.911 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36608 | min safe capacity: 1.1172 | + [2026-09-21 07:58:47] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 126.6 | throughput per GPU (TFLOP/s/GPU): 279.4 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.808459E+01 | loss scale: 1.0 | grad norm: 108.268 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 | + [2026-09-21 07:58:48] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 118.5 | throughput per GPU (TFLOP/s/GPU): 298.4 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.491727E+01 | loss scale: 1.0 | grad norm: 83.473 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 36736 | min safe capacity: 1.1211 | +[after training is done] datetime: 2026-09-21 07:58:48 +[rank0]:[W921 07:58:49.314872865 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0921 07:58:49.696000 556752 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0921 07:58:49.696000 556752 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 556857 closing signal SIGTERM +W0921 07:58:49.697000 556752 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 556858 closing signal SIGTERM +W0921 07:58:49.698000 556752 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 556859 closing signal SIGTERM +W0921 07:58:49.699000 556752 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 556860 closing signal SIGTERM +W0921 07:58:49.701000 556752 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 556861 closing signal SIGTERM +W0921 07:58:49.702000 556752 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 556862 closing signal SIGTERM +W0921 07:58:49.703000 556752 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 556863 closing signal SIGTERM +W0921 07:58:49.703000 556752 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 556864 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 556752 got signal: 15 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..515de57e1d9300a93ef20b438c4e58b32830b719 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/mem_samples.csv @@ -0,0 +1,224 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 809 +1, 783 +2, 791 +3, 795 +4, 787 +5, 785 +6, 791 +7, 791 +0, 1079 +1, 1075 +2, 1077 +3, 1077 +4, 1079 +5, 1079 +6, 1077 +7, 1079 +0, 1927 +1, 1927 +2, 1927 +3, 1927 +4, 1927 +5, 1927 +6, 1927 +7, 1927 +0, 3825 +1, 3803 +2, 3815 +3, 3833 +4, 3839 +5, 3835 +6, 3829 +7, 3827 +0, 50559 +1, 50559 +2, 50559 +3, 50559 +4, 50559 +5, 50559 +6, 50559 +7, 50559 +0, 55537 +1, 55537 +2, 55537 +3, 55537 +4, 55537 +5, 55537 +6, 55537 +7, 55537 +0, 60285 +1, 60277 +2, 60267 +3, 60279 +4, 60265 +5, 60243 +6, 60283 +7, 60285 +0, 61249 +1, 61251 +2, 61249 +3, 61249 +4, 61249 +5, 61249 +6, 61249 +7, 61249 +0, 61249 +1, 61251 +2, 61249 +3, 61249 +4, 61249 +5, 61249 +6, 61249 +7, 61249 +0, 61445 +1, 61447 +2, 61445 +3, 61445 +4, 61445 +5, 61445 +6, 61445 +7, 61445 +0, 64405 +1, 64423 +2, 64397 +3, 64347 +4, 64459 +5, 64419 +6, 64421 +7, 64439 +0, 65423 +1, 65435 +2, 65415 +3, 65381 +4, 65511 +5, 65483 +6, 65455 +7, 65479 +0, 65905 +1, 65923 +2, 65897 +3, 65847 +4, 65959 +5, 65919 +6, 65921 +7, 65939 +0, 66159 +1, 66179 +2, 66149 +3, 66093 +4, 66221 +5, 66175 +6, 66177 +7, 66197 +0, 66163 +1, 66181 +2, 66153 +3, 66097 +4, 66225 +5, 66179 +6, 66181 +7, 66201 +0, 68543 +1, 68523 +2, 68529 +3, 68469 +4, 68617 +5, 68535 +6, 68583 +7, 68639 +0, 69443 +1, 69497 +2, 69477 +3, 69373 +4, 69529 +5, 69495 +6, 69437 +7, 69471 +0, 75981 +1, 75977 +2, 75919 +3, 75889 +4, 76127 +5, 75999 +6, 75989 +7, 76027 +0, 89651 +1, 89685 +2, 89643 +3, 89723 +4, 89727 +5, 89673 +6, 89711 +7, 89699 +0, 89075 +1, 89045 +2, 88805 +3, 89147 +4, 88825 +5, 88515 +6, 88815 +7, 89123 +0, 73173 +1, 72951 +2, 73491 +3, 73309 +4, 73575 +5, 73583 +6, 73365 +7, 73935 +0, 71425 +1, 71271 +2, 71811 +3, 71629 +4, 71895 +5, 71579 +6, 71943 +7, 72513 +0, 324 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 68 +7, 70833 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260921T075953365187Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260921T075953365187Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aa28dde4130202105e507f4d047501ee4def1574 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260921T075953365187Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624399872, + "active_bytes_current_delta": 21596953600, + "active_bytes_peak": 51624402944, + "active_bytes_peak_delta": 21538236416, + "after": { + "active_bytes_current": 51624399872, + "active_bytes_peak": 51624402944, + "allocated_bytes_current": 51624399872, + "allocated_bytes_peak": 51624402944, + "device": 0, + "inactive_split_bytes_current": 689056768, + "inactive_split_bytes_peak": 2915038208, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 57237569536, + "reserved_bytes_peak": 57237569536, + "segment_count_current": 630, + "segment_count_peak": 630, + "timestamp_utc": "2026-09-21T07:59:53.364654Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624399872, + "allocated_bytes_current_delta": 21596953600, + "allocated_bytes_peak": 51624402944, + "allocated_bytes_peak_delta": 21538236416, + "before": { + "active_bytes_current": 30027446272, + "active_bytes_peak": 30086166528, + "allocated_bytes_current": 30027446272, + "allocated_bytes_peak": 30086166528, + "device": 0, + "inactive_split_bytes_current": 175736832, + "inactive_split_bytes_peak": 762939392, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 35888562176, + "reserved_bytes_peak": 35888562176, + "segment_count_current": 243, + "segment_count_peak": 243, + "timestamp_utc": "2026-09-21T07:59:22.956444Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 689056768, + "inactive_split_bytes_current_delta": 513319936, + "inactive_split_bytes_peak": 2915038208, + "inactive_split_bytes_peak_delta": 2152098816, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 57237569536, + "reserved_bytes_current_delta": 21349007360, + "reserved_bytes_peak": 57237569536, + "reserved_bytes_peak_delta": 21349007360, + "segment_count_current": 630, + "segment_count_current_delta": 387, + "segment_count_peak": 630, + "segment_count_peak_delta": 387, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260921T075953365187Z.pickle", + "timestamp_utc": "2026-09-21T07:59:53.380548Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260921T075953365187Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260921T075953365187Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fe530e9ff847f88c9268ceb258af1b8e4721c0f9 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260921T075953365187Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b72c412dbbc8f12e83afeb83221632f9be7e1ed6391b86ddb7ab587660079f8e +size 746089 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260921T075954665739Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260921T075954665739Z.json new file mode 100644 index 0000000000000000000000000000000000000000..498a7237e6771aec28093936308250cb469f8a12 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260921T075954665739Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624399872, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68336138240, + "active_bytes_peak_delta": 16711735296, + "after": { + "active_bytes_current": 51624399872, + "active_bytes_peak": 68336138240, + "allocated_bytes_current": 51624399872, + "allocated_bytes_peak": 68336138240, + "device": 0, + "inactive_split_bytes_current": 689056768, + "inactive_split_bytes_peak": 4260556800, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 71380762624, + "reserved_bytes_peak": 71380762624, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:54.665162Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624399872, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68336138240, + "allocated_bytes_peak_delta": 16711735296, + "before": { + "active_bytes_current": 51624399872, + "active_bytes_peak": 51624402944, + "allocated_bytes_current": 51624399872, + "allocated_bytes_peak": 51624402944, + "device": 0, + "inactive_split_bytes_current": 689056768, + "inactive_split_bytes_peak": 2915038208, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 57237569536, + "reserved_bytes_peak": 57237569536, + "segment_count_current": 630, + "segment_count_peak": 630, + "timestamp_utc": "2026-09-21T07:59:53.382764Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 689056768, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4260556800, + "inactive_split_bytes_peak_delta": 1345518592, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 71380762624, + "reserved_bytes_current_delta": 14143193088, + "reserved_bytes_peak": 71380762624, + "reserved_bytes_peak_delta": 14143193088, + "segment_count_current": 692, + "segment_count_current_delta": 62, + "segment_count_peak": 692, + "segment_count_peak_delta": 62, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260921T075954665739Z.pickle", + "timestamp_utc": "2026-09-21T07:59:54.678743Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260921T075954665739Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260921T075954665739Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..023867612f50928fa34f568065ba54ddab166a59 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260921T075954665739Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bfeae6685d345824c627db5f7107678599ffda2198c8de2ff4f73e998360390 +size 632262 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260921T075955731818Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260921T075955731818Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fb154e5f92c07773941ffc1ef387e78d196c203b --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260921T075955731818Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624399872, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68336138240, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51624399872, + "active_bytes_peak": 68336138240, + "allocated_bytes_current": 51624399872, + "allocated_bytes_peak": 68336138240, + "device": 0, + "inactive_split_bytes_current": 689056768, + "inactive_split_bytes_peak": 4386648064, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 71380762624, + "reserved_bytes_peak": 71380762624, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:55.731361Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624399872, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68336138240, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51624399872, + "active_bytes_peak": 68336138240, + "allocated_bytes_current": 51624399872, + "allocated_bytes_peak": 68336138240, + "device": 0, + "inactive_split_bytes_current": 689056768, + "inactive_split_bytes_peak": 4260556800, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 71380762624, + "reserved_bytes_peak": 71380762624, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:54.680635Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 689056768, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4386648064, + "inactive_split_bytes_peak_delta": 126091264, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 71380762624, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71380762624, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 692, + "segment_count_current_delta": 0, + "segment_count_peak": 692, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260921T075955731818Z.pickle", + "timestamp_utc": "2026-09-21T07:59:55.743303Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260921T075955731818Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260921T075955731818Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ef48196c1cdc8722a6030af321ef4db2cdd75284 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260921T075955731818Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f53376bf64d8121c5a8b2961c5d7bffa65f974406a134556fb8870ee36c8d734 +size 626426 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260921T075955883637Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260921T075955883637Z.json new file mode 100644 index 0000000000000000000000000000000000000000..55c5db46da870f6b56ac9ef753ae634bf32b75f1 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260921T075955883637Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624399872, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68336662528, + "active_bytes_peak_delta": 524288, + "after": { + "active_bytes_current": 51624399872, + "active_bytes_peak": 68336662528, + "allocated_bytes_current": 51624399872, + "allocated_bytes_peak": 68336662528, + "device": 0, + "inactive_split_bytes_current": 689056768, + "inactive_split_bytes_peak": 4386648064, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 71380762624, + "reserved_bytes_peak": 71380762624, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:55.883205Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624399872, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68336662528, + "allocated_bytes_peak_delta": 524288, + "before": { + "active_bytes_current": 51624399872, + "active_bytes_peak": 68336138240, + "allocated_bytes_current": 51624399872, + "allocated_bytes_peak": 68336138240, + "device": 0, + "inactive_split_bytes_current": 689056768, + "inactive_split_bytes_peak": 4386648064, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 71380762624, + "reserved_bytes_peak": 71380762624, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:55.745022Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 689056768, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4386648064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 71380762624, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71380762624, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 692, + "segment_count_current_delta": 0, + "segment_count_peak": 692, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260921T075955883637Z.pickle", + "timestamp_utc": "2026-09-21T07:59:55.894763Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260921T075955883637Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260921T075955883637Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..907beb1fbde89a80fd38af2a57a878f490fc85b4 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260921T075955883637Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eca45ca28f83c7b3e16aeee54ed038f8664f487993753d72e940c3f8e3f6ace7 +size 626426 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260921T075956036514Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260921T075956036514Z.json new file mode 100644 index 0000000000000000000000000000000000000000..875c6993e4a8d886d2acc48eab696ab29e57d6b1 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260921T075956036514Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624399872, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68336662528, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51624399872, + "active_bytes_peak": 68336662528, + "allocated_bytes_current": 51624399872, + "allocated_bytes_peak": 68336662528, + "device": 0, + "inactive_split_bytes_current": 689056768, + "inactive_split_bytes_peak": 4445106176, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 71380762624, + "reserved_bytes_peak": 71380762624, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:56.036053Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624399872, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68336662528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51624399872, + "active_bytes_peak": 68336662528, + "allocated_bytes_current": 51624399872, + "allocated_bytes_peak": 68336662528, + "device": 0, + "inactive_split_bytes_current": 689056768, + "inactive_split_bytes_peak": 4386648064, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 71380762624, + "reserved_bytes_peak": 71380762624, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:55.896427Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 689056768, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4445106176, + "inactive_split_bytes_peak_delta": 58458112, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 71380762624, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71380762624, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 692, + "segment_count_current_delta": 0, + "segment_count_peak": 692, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260921T075956036514Z.pickle", + "timestamp_utc": "2026-09-21T07:59:56.047714Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260921T075956036514Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260921T075956036514Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5e58ff4344cd2185e522af3cf44b607d9e9f4de2 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260921T075956036514Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b849797a8c7909bfa527a1594f74ebb14f37a1e708328dcf5dc8d3320b76de8 +size 626426 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260921T075953365067Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260921T075953365067Z.json new file mode 100644 index 0000000000000000000000000000000000000000..469e94334eb1aa22c1e9e41ab8de31237781a2f3 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260921T075953365067Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624832000, + "active_bytes_current_delta": 21597385728, + "active_bytes_peak": 51624835072, + "active_bytes_peak_delta": 21538668544, + "after": { + "active_bytes_current": 51624832000, + "active_bytes_peak": 51624835072, + "allocated_bytes_current": 51624832000, + "allocated_bytes_peak": 51624835072, + "device": 1, + "inactive_split_bytes_current": 699110400, + "inactive_split_bytes_peak": 2860219904, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 57256443904, + "reserved_bytes_peak": 57256443904, + "segment_count_current": 631, + "segment_count_peak": 631, + "timestamp_utc": "2026-09-21T07:59:53.364543Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624832000, + "allocated_bytes_current_delta": 21597385728, + "allocated_bytes_peak": 51624835072, + "allocated_bytes_peak_delta": 21538668544, + "before": { + "active_bytes_current": 30027446272, + "active_bytes_peak": 30086166528, + "allocated_bytes_current": 30027446272, + "allocated_bytes_peak": 30086166528, + "device": 1, + "inactive_split_bytes_current": 175736832, + "inactive_split_bytes_peak": 762939392, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 35888562176, + "reserved_bytes_peak": 35888562176, + "segment_count_current": 243, + "segment_count_peak": 243, + "timestamp_utc": "2026-09-21T07:59:22.956330Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 699110400, + "inactive_split_bytes_current_delta": 523373568, + "inactive_split_bytes_peak": 2860219904, + "inactive_split_bytes_peak_delta": 2097280512, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 57256443904, + "reserved_bytes_current_delta": 21367881728, + "reserved_bytes_peak": 57256443904, + "reserved_bytes_peak_delta": 21367881728, + "segment_count_current": 631, + "segment_count_current_delta": 388, + "segment_count_peak": 631, + "segment_count_peak_delta": 388, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260921T075953365067Z.pickle", + "timestamp_utc": "2026-09-21T07:59:53.379796Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260921T075953365067Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260921T075953365067Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..732abb455d299e5677923fd7323d87d4fd7a1afa --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260921T075953365067Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59553e59c0fab482eb6881aee9b9eef092ed49965cb9580a9a57af1c70e36744 +size 745838 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260921T075954665723Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260921T075954665723Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a0c450e06818796f9d5955683b05b4393bd989a0 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260921T075954665723Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624832000, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68339226112, + "active_bytes_peak_delta": 16714391040, + "after": { + "active_bytes_current": 51624832000, + "active_bytes_peak": 68339226112, + "allocated_bytes_current": 51624832000, + "allocated_bytes_peak": 68339226112, + "device": 1, + "inactive_split_bytes_current": 699110400, + "inactive_split_bytes_peak": 4295976448, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 71416414208, + "reserved_bytes_peak": 71416414208, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-09-21T07:59:54.665179Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624832000, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68339226112, + "allocated_bytes_peak_delta": 16714391040, + "before": { + "active_bytes_current": 51624832000, + "active_bytes_peak": 51624835072, + "allocated_bytes_current": 51624832000, + "allocated_bytes_peak": 51624835072, + "device": 1, + "inactive_split_bytes_current": 699110400, + "inactive_split_bytes_peak": 2860219904, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 57256443904, + "reserved_bytes_peak": 57256443904, + "segment_count_current": 631, + "segment_count_peak": 631, + "timestamp_utc": "2026-09-21T07:59:53.381952Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 699110400, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4295976448, + "inactive_split_bytes_peak_delta": 1435756544, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 71416414208, + "reserved_bytes_current_delta": 14159970304, + "reserved_bytes_peak": 71416414208, + "reserved_bytes_peak_delta": 14159970304, + "segment_count_current": 693, + "segment_count_current_delta": 62, + "segment_count_peak": 693, + "segment_count_peak_delta": 62, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260921T075954665723Z.pickle", + "timestamp_utc": "2026-09-21T07:59:54.678781Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260921T075954665723Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260921T075954665723Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e7fa10044fc3aaa026b56817b4b3956e090ce54a --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260921T075954665723Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5780f6d2036414bb6ea7d3adc652af4ff64e41d7cd739326b6e9d1fa21f9f20c +size 632275 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260921T075955731809Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260921T075955731809Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4b24656a43608f9e2f61cfa78f35a178929659ee --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260921T075955731809Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624832000, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68339893760, + "active_bytes_peak_delta": 667648, + "after": { + "active_bytes_current": 51624832000, + "active_bytes_peak": 68339893760, + "allocated_bytes_current": 51624832000, + "allocated_bytes_peak": 68339893760, + "device": 1, + "inactive_split_bytes_current": 699110400, + "inactive_split_bytes_peak": 4420494848, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 71416414208, + "reserved_bytes_peak": 71416414208, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-09-21T07:59:55.731364Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624832000, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68339893760, + "allocated_bytes_peak_delta": 667648, + "before": { + "active_bytes_current": 51624832000, + "active_bytes_peak": 68339226112, + "allocated_bytes_current": 51624832000, + "allocated_bytes_peak": 68339226112, + "device": 1, + "inactive_split_bytes_current": 699110400, + "inactive_split_bytes_peak": 4295976448, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 71416414208, + "reserved_bytes_peak": 71416414208, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-09-21T07:59:54.680719Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 699110400, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4420494848, + "inactive_split_bytes_peak_delta": 124518400, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 71416414208, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71416414208, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 693, + "segment_count_current_delta": 0, + "segment_count_peak": 693, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260921T075955731809Z.pickle", + "timestamp_utc": "2026-09-21T07:59:55.743033Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260921T075955731809Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260921T075955731809Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..60ff96a6a7fa24e8a9b3aa77155194e42f583802 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260921T075955731809Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55afa0e02bf9285530d49f25e24971c9785946ed4b676e99c207977b2a27ecdc +size 626461 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260921T075955883637Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260921T075955883637Z.json new file mode 100644 index 0000000000000000000000000000000000000000..35a10f6dd04f2eac04ccbac8bfa5173dd6bd0e9a --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260921T075955883637Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624832000, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68339893760, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51624832000, + "active_bytes_peak": 68339893760, + "allocated_bytes_current": 51624832000, + "allocated_bytes_peak": 68339893760, + "device": 1, + "inactive_split_bytes_current": 699110400, + "inactive_split_bytes_peak": 4420494848, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 71416414208, + "reserved_bytes_peak": 71416414208, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-09-21T07:59:55.883194Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624832000, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68339893760, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51624832000, + "active_bytes_peak": 68339893760, + "allocated_bytes_current": 51624832000, + "allocated_bytes_peak": 68339893760, + "device": 1, + "inactive_split_bytes_current": 699110400, + "inactive_split_bytes_peak": 4420494848, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 71416414208, + "reserved_bytes_peak": 71416414208, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-09-21T07:59:55.744773Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 699110400, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4420494848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 71416414208, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71416414208, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 693, + "segment_count_current_delta": 0, + "segment_count_peak": 693, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260921T075955883637Z.pickle", + "timestamp_utc": "2026-09-21T07:59:55.894415Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260921T075955883637Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260921T075955883637Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f853dfce8bc431fd1b0634642b87313172ade558 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260921T075955883637Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6772057494b63abac9ab302c77e2f90b46b46d6cf9634c7bed5e447a59d6f517 +size 626464 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260921T075956036496Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260921T075956036496Z.json new file mode 100644 index 0000000000000000000000000000000000000000..36e6acd00267c4b233d2627bf1517f2b14bb503d --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260921T075956036496Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624832000, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68339893760, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51624832000, + "active_bytes_peak": 68339893760, + "allocated_bytes_current": 51624832000, + "allocated_bytes_peak": 68339893760, + "device": 1, + "inactive_split_bytes_current": 699110400, + "inactive_split_bytes_peak": 4420494848, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 71416414208, + "reserved_bytes_peak": 71416414208, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-09-21T07:59:56.036051Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624832000, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68339893760, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51624832000, + "active_bytes_peak": 68339893760, + "allocated_bytes_current": 51624832000, + "allocated_bytes_peak": 68339893760, + "device": 1, + "inactive_split_bytes_current": 699110400, + "inactive_split_bytes_peak": 4420494848, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 71416414208, + "reserved_bytes_peak": 71416414208, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-09-21T07:59:55.896033Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 699110400, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4420494848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 71416414208, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71416414208, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 693, + "segment_count_current_delta": 0, + "segment_count_peak": 693, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260921T075956036496Z.pickle", + "timestamp_utc": "2026-09-21T07:59:56.047408Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260921T075956036496Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260921T075956036496Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..df974e9aa2eac925843df9f6a0da3a5b8e53c0eb --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260921T075956036496Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:675abf83cd8063162ca3ddc6595a08fbf455928d4976b7b3fa8bfb99a5b89faa +size 626461 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260921T075953364965Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260921T075953364965Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d76989d5298b9841d403813ecb2fd492331deff0 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260921T075953364965Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51628065280, + "active_bytes_current_delta": 21599633408, + "active_bytes_peak": 51628068352, + "active_bytes_peak_delta": 21540916224, + "after": { + "active_bytes_current": 51628065280, + "active_bytes_peak": 51628068352, + "allocated_bytes_current": 51628065280, + "allocated_bytes_peak": 51628068352, + "device": 2, + "inactive_split_bytes_current": 632962560, + "inactive_split_bytes_peak": 2854405632, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 57227083776, + "reserved_bytes_peak": 57227083776, + "segment_count_current": 630, + "segment_count_peak": 630, + "timestamp_utc": "2026-09-21T07:59:53.364421Z", + "world_size": 8 + }, + "allocated_bytes_current": 51628065280, + "allocated_bytes_current_delta": 21599633408, + "allocated_bytes_peak": 51628068352, + "allocated_bytes_peak_delta": 21540916224, + "before": { + "active_bytes_current": 30028431872, + "active_bytes_peak": 30087152128, + "allocated_bytes_current": 30028431872, + "allocated_bytes_peak": 30087152128, + "device": 2, + "inactive_split_bytes_current": 174751232, + "inactive_split_bytes_peak": 761953792, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 35888562176, + "reserved_bytes_peak": 35888562176, + "segment_count_current": 243, + "segment_count_peak": 243, + "timestamp_utc": "2026-09-21T07:59:22.956339Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 632962560, + "inactive_split_bytes_current_delta": 458211328, + "inactive_split_bytes_peak": 2854405632, + "inactive_split_bytes_peak_delta": 2092451840, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 57227083776, + "reserved_bytes_current_delta": 21338521600, + "reserved_bytes_peak": 57227083776, + "reserved_bytes_peak_delta": 21338521600, + "segment_count_current": 630, + "segment_count_current_delta": 387, + "segment_count_peak": 630, + "segment_count_peak_delta": 387, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260921T075953364965Z.pickle", + "timestamp_utc": "2026-09-21T07:59:53.379343Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260921T075953364965Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260921T075953364965Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8ddad331fbbe97ecd4d0405f091cc26b296e31a3 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260921T075953364965Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90c5899b6968eaad176082b57e51fa6790bb4e7cf1defb45cf9d75383ee6f82d +size 749816 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260921T075954665403Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260921T075954665403Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8597a3c15f597e58222819a1a9095add5cdbddbd --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260921T075954665403Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51628065280, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68342117376, + "active_bytes_peak_delta": 16714049024, + "after": { + "active_bytes_current": 51628065280, + "active_bytes_peak": 68342117376, + "allocated_bytes_current": 51628065280, + "allocated_bytes_peak": 68342117376, + "device": 2, + "inactive_split_bytes_current": 632962560, + "inactive_split_bytes_peak": 4203240448, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 71372374016, + "reserved_bytes_peak": 71372374016, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:54.664995Z", + "world_size": 8 + }, + "allocated_bytes_current": 51628065280, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68342117376, + "allocated_bytes_peak_delta": 16714049024, + "before": { + "active_bytes_current": 51628065280, + "active_bytes_peak": 51628068352, + "allocated_bytes_current": 51628065280, + "allocated_bytes_peak": 51628068352, + "device": 2, + "inactive_split_bytes_current": 632962560, + "inactive_split_bytes_peak": 2854405632, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 57227083776, + "reserved_bytes_peak": 57227083776, + "segment_count_current": 630, + "segment_count_peak": 630, + "timestamp_utc": "2026-09-21T07:59:53.381487Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 632962560, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4203240448, + "inactive_split_bytes_peak_delta": 1348834816, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 71372374016, + "reserved_bytes_current_delta": 14145290240, + "reserved_bytes_peak": 71372374016, + "reserved_bytes_peak_delta": 14145290240, + "segment_count_current": 692, + "segment_count_current_delta": 62, + "segment_count_peak": 692, + "segment_count_peak_delta": 62, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260921T075954665403Z.pickle", + "timestamp_utc": "2026-09-21T07:59:54.677740Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260921T075954665403Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260921T075954665403Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..04909949a136dfeaaabfd8c6a0ed4f9ec0326bde --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260921T075954665403Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6b8e517be502210d22b4e0e9a602f5b72b137babc3179f9ac45476437325404 +size 631596 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260921T075955731618Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260921T075955731618Z.json new file mode 100644 index 0000000000000000000000000000000000000000..def84fb5f19ec852be36d165419c9c0405869e84 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260921T075955731618Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51628065280, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68342117376, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51628065280, + "active_bytes_peak": 68342117376, + "allocated_bytes_current": 51628065280, + "allocated_bytes_peak": 68342117376, + "device": 2, + "inactive_split_bytes_current": 632962560, + "inactive_split_bytes_peak": 4302330880, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 71372374016, + "reserved_bytes_peak": 71372374016, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:55.731213Z", + "world_size": 8 + }, + "allocated_bytes_current": 51628065280, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68342117376, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51628065280, + "active_bytes_peak": 68342117376, + "allocated_bytes_current": 51628065280, + "allocated_bytes_peak": 68342117376, + "device": 2, + "inactive_split_bytes_current": 632962560, + "inactive_split_bytes_peak": 4203240448, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 71372374016, + "reserved_bytes_peak": 71372374016, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:54.679535Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 632962560, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4302330880, + "inactive_split_bytes_peak_delta": 99090432, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 71372374016, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71372374016, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 692, + "segment_count_current_delta": 0, + "segment_count_peak": 692, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260921T075955731618Z.pickle", + "timestamp_utc": "2026-09-21T07:59:55.742369Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260921T075955731618Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260921T075955731618Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e0c419ae550a2ab3a2eddd208179aa981798fa62 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260921T075955731618Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcce0a4d339e35411e46894e70ee128dd868c708a45b3a2ae8bbc3a2ee5f8a9e +size 625148 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260921T075955883471Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260921T075955883471Z.json new file mode 100644 index 0000000000000000000000000000000000000000..96eee185b5c46f5c9e7bf185c65948274e8fd590 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260921T075955883471Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51628065280, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68342117376, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51628065280, + "active_bytes_peak": 68342117376, + "allocated_bytes_current": 51628065280, + "allocated_bytes_peak": 68342117376, + "device": 2, + "inactive_split_bytes_current": 632962560, + "inactive_split_bytes_peak": 4302330880, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 71372374016, + "reserved_bytes_peak": 71372374016, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:55.883082Z", + "world_size": 8 + }, + "allocated_bytes_current": 51628065280, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68342117376, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51628065280, + "active_bytes_peak": 68342117376, + "allocated_bytes_current": 51628065280, + "allocated_bytes_peak": 68342117376, + "device": 2, + "inactive_split_bytes_current": 632962560, + "inactive_split_bytes_peak": 4302330880, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 71372374016, + "reserved_bytes_peak": 71372374016, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:55.744018Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 632962560, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4302330880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 71372374016, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71372374016, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 692, + "segment_count_current_delta": 0, + "segment_count_peak": 692, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260921T075955883471Z.pickle", + "timestamp_utc": "2026-09-21T07:59:55.893911Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260921T075955883471Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260921T075955883471Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4dc7867322400763e98c7140d796da3671d97def --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260921T075955883471Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98619934802cbd90af14186b5a88b58cb1dfed11eb7457156c8658eb296606b6 +size 625116 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260921T075956036341Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260921T075956036341Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c0f51f799906131015f2db79af1a69c069fbf5cb --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260921T075956036341Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51628065280, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68342117376, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51628065280, + "active_bytes_peak": 68342117376, + "allocated_bytes_current": 51628065280, + "allocated_bytes_peak": 68342117376, + "device": 2, + "inactive_split_bytes_current": 632962560, + "inactive_split_bytes_peak": 4302330880, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 71372374016, + "reserved_bytes_peak": 71372374016, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:56.035953Z", + "world_size": 8 + }, + "allocated_bytes_current": 51628065280, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68342117376, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51628065280, + "active_bytes_peak": 68342117376, + "allocated_bytes_current": 51628065280, + "allocated_bytes_peak": 68342117376, + "device": 2, + "inactive_split_bytes_current": 632962560, + "inactive_split_bytes_peak": 4302330880, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 71372374016, + "reserved_bytes_peak": 71372374016, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:55.895521Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 632962560, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4302330880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 71372374016, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71372374016, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 692, + "segment_count_current_delta": 0, + "segment_count_peak": 692, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260921T075956036341Z.pickle", + "timestamp_utc": "2026-09-21T07:59:56.046889Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260921T075956036341Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260921T075956036341Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5ba21b3e60922084f402b68303f68c53fbacb513 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260921T075956036341Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86cd86db4dd36fe42f5625eacdd790bc3c779bf7e63e47b1e766a38a74effe21 +size 625148 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260921T075953365044Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260921T075953365044Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ed4c7b5b06068a94d52cb0f28d300eeb9ae278ce --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260921T075953365044Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624433152, + "active_bytes_current_delta": 21596986880, + "active_bytes_peak": 51624436224, + "active_bytes_peak_delta": 21538269696, + "after": { + "active_bytes_current": 51624433152, + "active_bytes_peak": 51624436224, + "allocated_bytes_current": 51624433152, + "allocated_bytes_peak": 51624436224, + "device": 3, + "inactive_split_bytes_current": 665954816, + "inactive_split_bytes_peak": 2895078400, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 57168363520, + "reserved_bytes_peak": 57168363520, + "segment_count_current": 627, + "segment_count_peak": 627, + "timestamp_utc": "2026-09-21T07:59:53.364502Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624433152, + "allocated_bytes_current_delta": 21596986880, + "allocated_bytes_peak": 51624436224, + "allocated_bytes_peak_delta": 21538269696, + "before": { + "active_bytes_current": 30027446272, + "active_bytes_peak": 30086166528, + "allocated_bytes_current": 30027446272, + "allocated_bytes_peak": 30086166528, + "device": 3, + "inactive_split_bytes_current": 175736832, + "inactive_split_bytes_peak": 762939392, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 35888562176, + "reserved_bytes_peak": 35888562176, + "segment_count_current": 243, + "segment_count_peak": 243, + "timestamp_utc": "2026-09-21T07:59:22.956364Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 665954816, + "inactive_split_bytes_current_delta": 490217984, + "inactive_split_bytes_peak": 2895078400, + "inactive_split_bytes_peak_delta": 2132139008, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 57168363520, + "reserved_bytes_current_delta": 21279801344, + "reserved_bytes_peak": 57168363520, + "reserved_bytes_peak_delta": 21279801344, + "segment_count_current": 627, + "segment_count_current_delta": 384, + "segment_count_peak": 627, + "segment_count_peak_delta": 384, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260921T075953365044Z.pickle", + "timestamp_utc": "2026-09-21T07:59:53.379760Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260921T075953365044Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260921T075953365044Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..059beb97675164e928d0c1904a9966bc5fd3116d --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260921T075953365044Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14a5ae89e553e931ca07e07e245029ac4a03deac59c9702ab636bbd5ebacae4e +size 748596 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260921T075954665751Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260921T075954665751Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4ad8f6b50922d65479098ed5f43c494621d3e434 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260921T075954665751Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624433152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68337959936, + "active_bytes_peak_delta": 16713523712, + "after": { + "active_bytes_current": 51624433152, + "active_bytes_peak": 68337959936, + "allocated_bytes_current": 51624433152, + "allocated_bytes_peak": 68337959936, + "device": 3, + "inactive_split_bytes_current": 665954816, + "inactive_split_bytes_peak": 4324052992, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 71456260096, + "reserved_bytes_peak": 71456260096, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-09-21T07:59:54.665184Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624433152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68337959936, + "allocated_bytes_peak_delta": 16713523712, + "before": { + "active_bytes_current": 51624433152, + "active_bytes_peak": 51624436224, + "allocated_bytes_current": 51624433152, + "allocated_bytes_peak": 51624436224, + "device": 3, + "inactive_split_bytes_current": 665954816, + "inactive_split_bytes_peak": 2895078400, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 57168363520, + "reserved_bytes_peak": 57168363520, + "segment_count_current": 627, + "segment_count_peak": 627, + "timestamp_utc": "2026-09-21T07:59:53.381879Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 665954816, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4324052992, + "inactive_split_bytes_peak_delta": 1428974592, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 71456260096, + "reserved_bytes_current_delta": 14287896576, + "reserved_bytes_peak": 71456260096, + "reserved_bytes_peak_delta": 14287896576, + "segment_count_current": 693, + "segment_count_current_delta": 66, + "segment_count_peak": 693, + "segment_count_peak_delta": 66, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260921T075954665751Z.pickle", + "timestamp_utc": "2026-09-21T07:59:54.679091Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260921T075954665751Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260921T075954665751Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2268c729bdc8d6df74b33d527670bc0ab0d8381f --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260921T075954665751Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:360f7b953c7ac4dcbc991e71f48d907d48e7765a7b770cc2701f9cd5b349b04b +size 631400 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260921T075955731828Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260921T075955731828Z.json new file mode 100644 index 0000000000000000000000000000000000000000..32308955223c4accc1ebac0844499deb310b1260 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260921T075955731828Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624433152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68337959936, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51624433152, + "active_bytes_peak": 68337959936, + "allocated_bytes_current": 51624433152, + "allocated_bytes_peak": 68337959936, + "device": 3, + "inactive_split_bytes_current": 665954816, + "inactive_split_bytes_peak": 4358918144, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 71456260096, + "reserved_bytes_peak": 71456260096, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-09-21T07:59:55.731372Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624433152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68337959936, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51624433152, + "active_bytes_peak": 68337959936, + "allocated_bytes_current": 51624433152, + "allocated_bytes_peak": 68337959936, + "device": 3, + "inactive_split_bytes_current": 665954816, + "inactive_split_bytes_peak": 4324052992, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 71456260096, + "reserved_bytes_peak": 71456260096, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-09-21T07:59:54.680958Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 665954816, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4358918144, + "inactive_split_bytes_peak_delta": 34865152, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 71456260096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71456260096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 693, + "segment_count_current_delta": 0, + "segment_count_peak": 693, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260921T075955731828Z.pickle", + "timestamp_utc": "2026-09-21T07:59:55.743144Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260921T075955731828Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260921T075955731828Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2e9cd4dd9ce7d2ddded3edcb444726f332eb0915 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260921T075955731828Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df71286a7b6502f93089f15f52fb2254cc4fe97ba8085ffb8e085580c5c07ad7 +size 625879 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260921T075955883624Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260921T075955883624Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a3feb459b826809cc69627082c6dfc632c27143f --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260921T075955883624Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624433152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68337959936, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51624433152, + "active_bytes_peak": 68337959936, + "allocated_bytes_current": 51624433152, + "allocated_bytes_peak": 68337959936, + "device": 3, + "inactive_split_bytes_current": 665954816, + "inactive_split_bytes_peak": 4358918144, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 71456260096, + "reserved_bytes_peak": 71456260096, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-09-21T07:59:55.883204Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624433152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68337959936, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51624433152, + "active_bytes_peak": 68337959936, + "allocated_bytes_current": 51624433152, + "allocated_bytes_peak": 68337959936, + "device": 3, + "inactive_split_bytes_current": 665954816, + "inactive_split_bytes_peak": 4358918144, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 71456260096, + "reserved_bytes_peak": 71456260096, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-09-21T07:59:55.744839Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 665954816, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4358918144, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 71456260096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71456260096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 693, + "segment_count_current_delta": 0, + "segment_count_peak": 693, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260921T075955883624Z.pickle", + "timestamp_utc": "2026-09-21T07:59:55.894762Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260921T075955883624Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260921T075955883624Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..95db2b97a658bf4c8556914177e695985aef29cb --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260921T075955883624Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0358058566b92dc1622af1912cf575bcaa95a71690111f9367d361ab331fd11 +size 625847 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260921T075956036519Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260921T075956036519Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c5b18e76d72e2a32bcde07115e9c9158d55fa9e8 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260921T075956036519Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624433152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68337959936, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51624433152, + "active_bytes_peak": 68337959936, + "allocated_bytes_current": 51624433152, + "allocated_bytes_peak": 68337959936, + "device": 3, + "inactive_split_bytes_current": 665954816, + "inactive_split_bytes_peak": 4358918144, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 71456260096, + "reserved_bytes_peak": 71456260096, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-09-21T07:59:56.036056Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624433152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68337959936, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51624433152, + "active_bytes_peak": 68337959936, + "allocated_bytes_current": 51624433152, + "allocated_bytes_peak": 68337959936, + "device": 3, + "inactive_split_bytes_current": 665954816, + "inactive_split_bytes_peak": 4358918144, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 71456260096, + "reserved_bytes_peak": 71456260096, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-09-21T07:59:55.896379Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 665954816, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4358918144, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 71456260096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71456260096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 693, + "segment_count_current_delta": 0, + "segment_count_peak": 693, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260921T075956036519Z.pickle", + "timestamp_utc": "2026-09-21T07:59:56.047568Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260921T075956036519Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260921T075956036519Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1f8107d5328d007fd5a4ab0953cd01ca84165cda --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260921T075956036519Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f98d208179c88df143e24a39d14c7a7781c9c0950aa7a619bfb86ba396ad0f52 +size 625879 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260921T075953365072Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260921T075953365072Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9246480e015131a9a27fe2d2c9f99636becf248d --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260921T075953365072Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624433152, + "active_bytes_current_delta": 21596986880, + "active_bytes_peak": 51624436224, + "active_bytes_peak_delta": 21538269696, + "after": { + "active_bytes_current": 51624433152, + "active_bytes_peak": 51624436224, + "allocated_bytes_current": 51624433152, + "allocated_bytes_peak": 51624436224, + "device": 4, + "inactive_split_bytes_current": 829532672, + "inactive_split_bytes_peak": 2895078400, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 57420021760, + "reserved_bytes_peak": 57420021760, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-09-21T07:59:53.364521Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624433152, + "allocated_bytes_current_delta": 21596986880, + "allocated_bytes_peak": 51624436224, + "allocated_bytes_peak_delta": 21538269696, + "before": { + "active_bytes_current": 30027446272, + "active_bytes_peak": 30086166528, + "allocated_bytes_current": 30027446272, + "allocated_bytes_peak": 30086166528, + "device": 4, + "inactive_split_bytes_current": 175736832, + "inactive_split_bytes_peak": 762939392, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 35888562176, + "reserved_bytes_peak": 35888562176, + "segment_count_current": 243, + "segment_count_peak": 243, + "timestamp_utc": "2026-09-21T07:59:22.956764Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 829532672, + "inactive_split_bytes_current_delta": 653795840, + "inactive_split_bytes_peak": 2895078400, + "inactive_split_bytes_peak_delta": 2132139008, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 57420021760, + "reserved_bytes_current_delta": 21531459584, + "reserved_bytes_peak": 57420021760, + "reserved_bytes_peak_delta": 21531459584, + "segment_count_current": 634, + "segment_count_current_delta": 391, + "segment_count_peak": 634, + "segment_count_peak_delta": 391, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260921T075953365072Z.pickle", + "timestamp_utc": "2026-09-21T07:59:53.380587Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260921T075953365072Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260921T075953365072Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..616df2858ec3efee86c079877fdba74b3cb1d73d --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260921T075953365072Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aaaa5052c097cd49913368085b65a421ca9daaaa44d4ff4f2aecfea75d69737e +size 749789 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260921T075954665706Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260921T075954665706Z.json new file mode 100644 index 0000000000000000000000000000000000000000..63f77d15680556749870397d85f40b998c42ca8b --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260921T075954665706Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624433152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68337484800, + "active_bytes_peak_delta": 16713048576, + "after": { + "active_bytes_current": 51624433152, + "active_bytes_peak": 68337484800, + "allocated_bytes_current": 51624433152, + "allocated_bytes_peak": 68337484800, + "device": 4, + "inactive_split_bytes_current": 829532672, + "inactive_split_bytes_peak": 4370780160, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 71460454400, + "reserved_bytes_peak": 71460454400, + "segment_count_current": 689, + "segment_count_peak": 689, + "timestamp_utc": "2026-09-21T07:59:54.665164Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624433152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68337484800, + "allocated_bytes_peak_delta": 16713048576, + "before": { + "active_bytes_current": 51624433152, + "active_bytes_peak": 51624436224, + "allocated_bytes_current": 51624433152, + "allocated_bytes_peak": 51624436224, + "device": 4, + "inactive_split_bytes_current": 829532672, + "inactive_split_bytes_peak": 2895078400, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 57420021760, + "reserved_bytes_peak": 57420021760, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-09-21T07:59:53.382805Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 829532672, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4370780160, + "inactive_split_bytes_peak_delta": 1475701760, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 71460454400, + "reserved_bytes_current_delta": 14040432640, + "reserved_bytes_peak": 71460454400, + "reserved_bytes_peak_delta": 14040432640, + "segment_count_current": 689, + "segment_count_current_delta": 55, + "segment_count_peak": 689, + "segment_count_peak_delta": 55, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260921T075954665706Z.pickle", + "timestamp_utc": "2026-09-21T07:59:54.678946Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260921T075954665706Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260921T075954665706Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b23c98ce71b68521d51fee72069484cb915ea756 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260921T075954665706Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ad307cc4316c23e685b4f12d736745a1b3b630156d3c4d7c957124151d9f5db +size 631047 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260921T075955731914Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260921T075955731914Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e510a916a2957527dcf217c5544694d766b3e227 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260921T075955731914Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624433152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68337522176, + "active_bytes_peak_delta": 37376, + "after": { + "active_bytes_current": 51624433152, + "active_bytes_peak": 68337522176, + "allocated_bytes_current": 51624433152, + "allocated_bytes_peak": 68337522176, + "device": 4, + "inactive_split_bytes_current": 829532672, + "inactive_split_bytes_peak": 4495036416, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 71460454400, + "reserved_bytes_peak": 71460454400, + "segment_count_current": 689, + "segment_count_peak": 689, + "timestamp_utc": "2026-09-21T07:59:55.731394Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624433152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68337522176, + "allocated_bytes_peak_delta": 37376, + "before": { + "active_bytes_current": 51624433152, + "active_bytes_peak": 68337484800, + "allocated_bytes_current": 51624433152, + "allocated_bytes_peak": 68337484800, + "device": 4, + "inactive_split_bytes_current": 829532672, + "inactive_split_bytes_peak": 4370780160, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 71460454400, + "reserved_bytes_peak": 71460454400, + "segment_count_current": 689, + "segment_count_peak": 689, + "timestamp_utc": "2026-09-21T07:59:54.680766Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 829532672, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4495036416, + "inactive_split_bytes_peak_delta": 124256256, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 71460454400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71460454400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 689, + "segment_count_current_delta": 0, + "segment_count_peak": 689, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260921T075955731914Z.pickle", + "timestamp_utc": "2026-09-21T07:59:55.744634Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260921T075955731914Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260921T075955731914Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..29e29a9466cda26d0e185bd4b0d5bf2e4eb24ccb --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260921T075955731914Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf5eed0a452a9a3fb8981844d37a9c35220d2d60594ed01be8caa1ade82cbfb7 +size 625168 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260921T075955883625Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260921T075955883625Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b0d4e9ce2a5acd4318dd33cfb1fe0868a5a61197 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260921T075955883625Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624433152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68337522176, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51624433152, + "active_bytes_peak": 68337522176, + "allocated_bytes_current": 51624433152, + "allocated_bytes_peak": 68337522176, + "device": 4, + "inactive_split_bytes_current": 829532672, + "inactive_split_bytes_peak": 4495036416, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 71460454400, + "reserved_bytes_peak": 71460454400, + "segment_count_current": 689, + "segment_count_peak": 689, + "timestamp_utc": "2026-09-21T07:59:55.883206Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624433152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68337522176, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51624433152, + "active_bytes_peak": 68337522176, + "allocated_bytes_current": 51624433152, + "allocated_bytes_peak": 68337522176, + "device": 4, + "inactive_split_bytes_current": 829532672, + "inactive_split_bytes_peak": 4495036416, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 71460454400, + "reserved_bytes_peak": 71460454400, + "segment_count_current": 689, + "segment_count_peak": 689, + "timestamp_utc": "2026-09-21T07:59:55.746369Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 829532672, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4495036416, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 71460454400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71460454400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 689, + "segment_count_current_delta": 0, + "segment_count_peak": 689, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260921T075955883625Z.pickle", + "timestamp_utc": "2026-09-21T07:59:55.894956Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260921T075955883625Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260921T075955883625Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bb072f537db7aa20c0fd3bfcc1056764ebff0370 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260921T075955883625Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be17790587df8ea498a21f6a6a29a0f49475e202de128cca4e4ae9054063b701 +size 625136 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260921T075956036532Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260921T075956036532Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1a985474263e8a8fc938e4828fcf272d6d956c30 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260921T075956036532Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624433152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68337522176, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51624433152, + "active_bytes_peak": 68337522176, + "allocated_bytes_current": 51624433152, + "allocated_bytes_peak": 68337522176, + "device": 4, + "inactive_split_bytes_current": 829532672, + "inactive_split_bytes_peak": 4495036416, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 71460454400, + "reserved_bytes_peak": 71460454400, + "segment_count_current": 689, + "segment_count_peak": 689, + "timestamp_utc": "2026-09-21T07:59:56.036081Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624433152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68337522176, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51624433152, + "active_bytes_peak": 68337522176, + "allocated_bytes_current": 51624433152, + "allocated_bytes_peak": 68337522176, + "device": 4, + "inactive_split_bytes_current": 829532672, + "inactive_split_bytes_peak": 4495036416, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 71460454400, + "reserved_bytes_peak": 71460454400, + "segment_count_current": 689, + "segment_count_peak": 689, + "timestamp_utc": "2026-09-21T07:59:55.896598Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 829532672, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4495036416, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 71460454400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71460454400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 689, + "segment_count_current_delta": 0, + "segment_count_peak": 689, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260921T075956036532Z.pickle", + "timestamp_utc": "2026-09-21T07:59:56.047801Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260921T075956036532Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260921T075956036532Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dff3123f110db2153418026e4fb9ae960d13501d --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260921T075956036532Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43596c132a8b946d17b4b40736ee6e6c9be9096e93e77c5aa3d9953ce72a20da +size 625168 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260921T075953365096Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260921T075953365096Z.json new file mode 100644 index 0000000000000000000000000000000000000000..47fcabe7cc784727fe8e4928c28522b3c5dd9e77 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260921T075953365096Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51625993216, + "active_bytes_current_delta": 21598546944, + "active_bytes_peak": 51625996288, + "active_bytes_peak_delta": 21539829760, + "after": { + "active_bytes_current": 51625993216, + "active_bytes_peak": 51625996288, + "allocated_bytes_current": 51625993216, + "allocated_bytes_peak": 51625996288, + "device": 5, + "inactive_split_bytes_current": 691657728, + "inactive_split_bytes_peak": 3429096960, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 57254346752, + "reserved_bytes_peak": 57254346752, + "segment_count_current": 631, + "segment_count_peak": 631, + "timestamp_utc": "2026-09-21T07:59:53.364573Z", + "world_size": 8 + }, + "allocated_bytes_current": 51625993216, + "allocated_bytes_current_delta": 21598546944, + "allocated_bytes_peak": 51625996288, + "allocated_bytes_peak_delta": 21539829760, + "before": { + "active_bytes_current": 30027446272, + "active_bytes_peak": 30086166528, + "allocated_bytes_current": 30027446272, + "allocated_bytes_peak": 30086166528, + "device": 5, + "inactive_split_bytes_current": 175736832, + "inactive_split_bytes_peak": 762939392, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 35888562176, + "reserved_bytes_peak": 35888562176, + "segment_count_current": 243, + "segment_count_peak": 243, + "timestamp_utc": "2026-09-21T07:59:22.956402Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 691657728, + "inactive_split_bytes_current_delta": 515920896, + "inactive_split_bytes_peak": 3429096960, + "inactive_split_bytes_peak_delta": 2666157568, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 57254346752, + "reserved_bytes_current_delta": 21365784576, + "reserved_bytes_peak": 57254346752, + "reserved_bytes_peak_delta": 21365784576, + "segment_count_current": 631, + "segment_count_current_delta": 388, + "segment_count_peak": 631, + "segment_count_peak_delta": 388, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260921T075953365096Z.pickle", + "timestamp_utc": "2026-09-21T07:59:53.379889Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260921T075953365096Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260921T075953365096Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4e26570dcf7a8fc1646712aaeac168d0f72cfde9 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260921T075953365096Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1415b17a2313b9d0976a09e67808672e642a5de6fe6871ed66200d4b771d11b4 +size 745040 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260921T075954665693Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260921T075954665693Z.json new file mode 100644 index 0000000000000000000000000000000000000000..737b832770cb42d00912d377e082c95cb4fe1af0 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260921T075954665693Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51625993216, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68339206656, + "active_bytes_peak_delta": 16713210368, + "after": { + "active_bytes_current": 51625993216, + "active_bytes_peak": 68339206656, + "allocated_bytes_current": 51625993216, + "allocated_bytes_peak": 68339206656, + "device": 5, + "inactive_split_bytes_current": 691657728, + "inactive_split_bytes_peak": 4246970368, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 71403831296, + "reserved_bytes_peak": 71403831296, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:54.665168Z", + "world_size": 8 + }, + "allocated_bytes_current": 51625993216, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68339206656, + "allocated_bytes_peak_delta": 16713210368, + "before": { + "active_bytes_current": 51625993216, + "active_bytes_peak": 51625996288, + "allocated_bytes_current": 51625993216, + "allocated_bytes_peak": 51625996288, + "device": 5, + "inactive_split_bytes_current": 691657728, + "inactive_split_bytes_peak": 3429096960, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 57254346752, + "reserved_bytes_peak": 57254346752, + "segment_count_current": 631, + "segment_count_peak": 631, + "timestamp_utc": "2026-09-21T07:59:53.382039Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 691657728, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4246970368, + "inactive_split_bytes_peak_delta": 817873408, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 71403831296, + "reserved_bytes_current_delta": 14149484544, + "reserved_bytes_peak": 71403831296, + "reserved_bytes_peak_delta": 14149484544, + "segment_count_current": 692, + "segment_count_current_delta": 61, + "segment_count_peak": 692, + "segment_count_peak_delta": 61, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260921T075954665693Z.pickle", + "timestamp_utc": "2026-09-21T07:59:54.678413Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260921T075954665693Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260921T075954665693Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ffa89adb9c5ad95616748c8e0f20568e29cfa6dc --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260921T075954665693Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6f440e674feeeeaddab0aa45de8cd04bf566359a70437081f9e3860468ccae1 +size 625136 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260921T075955731909Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260921T075955731909Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8080f52fac84f72cf651482da5373262524c6a21 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260921T075955731909Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51625993216, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68339672064, + "active_bytes_peak_delta": 465408, + "after": { + "active_bytes_current": 51625993216, + "active_bytes_peak": 68339672064, + "allocated_bytes_current": 51625993216, + "allocated_bytes_peak": 68339672064, + "device": 5, + "inactive_split_bytes_current": 691657728, + "inactive_split_bytes_peak": 4406575616, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 71403831296, + "reserved_bytes_peak": 71403831296, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:55.731387Z", + "world_size": 8 + }, + "allocated_bytes_current": 51625993216, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68339672064, + "allocated_bytes_peak_delta": 465408, + "before": { + "active_bytes_current": 51625993216, + "active_bytes_peak": 68339206656, + "allocated_bytes_current": 51625993216, + "allocated_bytes_peak": 68339206656, + "device": 5, + "inactive_split_bytes_current": 691657728, + "inactive_split_bytes_peak": 4246970368, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 71403831296, + "reserved_bytes_peak": 71403831296, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:54.680216Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 691657728, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4406575616, + "inactive_split_bytes_peak_delta": 159605248, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 71403831296, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71403831296, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 692, + "segment_count_current_delta": 0, + "segment_count_peak": 692, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260921T075955731909Z.pickle", + "timestamp_utc": "2026-09-21T07:59:55.744532Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260921T075955731909Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260921T075955731909Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..460156e39df7f01699dd4c1a710f0b758befa93d --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260921T075955731909Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df04f325cd2ab498878a3adb75bbc96236853d29c8797408312fa1d879004dec +size 632584 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260921T075955883685Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260921T075955883685Z.json new file mode 100644 index 0000000000000000000000000000000000000000..99d6393b358e1488a99f27c5350ae6db16d02027 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260921T075955883685Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51625993216, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68339672064, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51625993216, + "active_bytes_peak": 68339672064, + "allocated_bytes_current": 51625993216, + "allocated_bytes_peak": 68339672064, + "device": 5, + "inactive_split_bytes_current": 691657728, + "inactive_split_bytes_peak": 4406575616, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 71403831296, + "reserved_bytes_peak": 71403831296, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:55.883246Z", + "world_size": 8 + }, + "allocated_bytes_current": 51625993216, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68339672064, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51625993216, + "active_bytes_peak": 68339672064, + "allocated_bytes_current": 51625993216, + "allocated_bytes_peak": 68339672064, + "device": 5, + "inactive_split_bytes_current": 691657728, + "inactive_split_bytes_peak": 4406575616, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 71403831296, + "reserved_bytes_peak": 71403831296, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:55.746295Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 691657728, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4406575616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 71403831296, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71403831296, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 692, + "segment_count_current_delta": 0, + "segment_count_peak": 692, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260921T075955883685Z.pickle", + "timestamp_utc": "2026-09-21T07:59:55.894782Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260921T075955883685Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260921T075955883685Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..37d73e1077f853b1f469abcd46007c1f27a1fa9b --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260921T075955883685Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bb05326b81a793d7eb5477b928a56f0387db903dcdc61e3e8e4aabeae745886 +size 625837 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260921T075956036520Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260921T075956036520Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2ede0a7516968c276ec1c317f389a6104b567777 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260921T075956036520Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51625993216, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68340306944, + "active_bytes_peak_delta": 634880, + "after": { + "active_bytes_current": 51625993216, + "active_bytes_peak": 68340306944, + "allocated_bytes_current": 51625993216, + "allocated_bytes_peak": 68340306944, + "device": 5, + "inactive_split_bytes_current": 691657728, + "inactive_split_bytes_peak": 4406575616, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 71403831296, + "reserved_bytes_peak": 71403831296, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:56.036070Z", + "world_size": 8 + }, + "allocated_bytes_current": 51625993216, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68340306944, + "allocated_bytes_peak_delta": 634880, + "before": { + "active_bytes_current": 51625993216, + "active_bytes_peak": 68339672064, + "allocated_bytes_current": 51625993216, + "allocated_bytes_peak": 68339672064, + "device": 5, + "inactive_split_bytes_current": 691657728, + "inactive_split_bytes_peak": 4406575616, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 71403831296, + "reserved_bytes_peak": 71403831296, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:55.896507Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 691657728, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4406575616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 71403831296, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71403831296, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 692, + "segment_count_current_delta": 0, + "segment_count_peak": 692, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260921T075956036520Z.pickle", + "timestamp_utc": "2026-09-21T07:59:56.047726Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260921T075956036520Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260921T075956036520Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..edf518e2f5b33325827af4dd9f5a560a8ec92c6c --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260921T075956036520Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd28de8f65986dee73f75e500a0de66fb961afb8298f5e901dd84b8b8b21f039 +size 625900 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260921T075953365075Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260921T075953365075Z.json new file mode 100644 index 0000000000000000000000000000000000000000..67be464ad91572c5495d969aeeb6e211f8dda332 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260921T075953365075Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624747520, + "active_bytes_current_delta": 21597301248, + "active_bytes_peak": 51624750592, + "active_bytes_peak_delta": 21538584064, + "after": { + "active_bytes_current": 51624747520, + "active_bytes_peak": 51624750592, + "allocated_bytes_current": 51624747520, + "allocated_bytes_peak": 51624750592, + "device": 6, + "inactive_split_bytes_current": 753720832, + "inactive_split_bytes_peak": 2917170688, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 57256443904, + "reserved_bytes_peak": 57256443904, + "segment_count_current": 631, + "segment_count_peak": 631, + "timestamp_utc": "2026-09-21T07:59:53.364528Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624747520, + "allocated_bytes_current_delta": 21597301248, + "allocated_bytes_peak": 51624750592, + "allocated_bytes_peak_delta": 21538584064, + "before": { + "active_bytes_current": 30027446272, + "active_bytes_peak": 30086166528, + "allocated_bytes_current": 30027446272, + "allocated_bytes_peak": 30086166528, + "device": 6, + "inactive_split_bytes_current": 175736832, + "inactive_split_bytes_peak": 762939392, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 35888562176, + "reserved_bytes_peak": 35888562176, + "segment_count_current": 243, + "segment_count_peak": 243, + "timestamp_utc": "2026-09-21T07:59:22.956326Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 753720832, + "inactive_split_bytes_current_delta": 577984000, + "inactive_split_bytes_peak": 2917170688, + "inactive_split_bytes_peak_delta": 2154231296, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 57256443904, + "reserved_bytes_current_delta": 21367881728, + "reserved_bytes_peak": 57256443904, + "reserved_bytes_peak_delta": 21367881728, + "segment_count_current": 631, + "segment_count_current_delta": 388, + "segment_count_peak": 631, + "segment_count_peak_delta": 388, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260921T075953365075Z.pickle", + "timestamp_utc": "2026-09-21T07:59:53.379789Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260921T075953365075Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260921T075953365075Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f1f744b166e36834798c3af1ea28657b8bd938c4 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260921T075953365075Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bc9136916f1418cab78f2e8c35e960b4412b0784d65e36dd871c20b207089d2 +size 745093 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260921T075954665701Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260921T075954665701Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5021456c904955c199e475cdb53436f2755a97fc --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260921T075954665701Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624747520, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68340472832, + "active_bytes_peak_delta": 16715722240, + "after": { + "active_bytes_current": 51624747520, + "active_bytes_peak": 68340472832, + "allocated_bytes_current": 51624747520, + "allocated_bytes_peak": 68340472832, + "device": 6, + "inactive_split_bytes_current": 753720832, + "inactive_split_bytes_peak": 4352139264, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 71443677184, + "reserved_bytes_peak": 71443677184, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:54.665142Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624747520, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68340472832, + "allocated_bytes_peak_delta": 16715722240, + "before": { + "active_bytes_current": 51624747520, + "active_bytes_peak": 51624750592, + "allocated_bytes_current": 51624747520, + "allocated_bytes_peak": 51624750592, + "device": 6, + "inactive_split_bytes_current": 753720832, + "inactive_split_bytes_peak": 2917170688, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 57256443904, + "reserved_bytes_peak": 57256443904, + "segment_count_current": 631, + "segment_count_peak": 631, + "timestamp_utc": "2026-09-21T07:59:53.381974Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 753720832, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4352139264, + "inactive_split_bytes_peak_delta": 1434968576, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 71443677184, + "reserved_bytes_current_delta": 14187233280, + "reserved_bytes_peak": 71443677184, + "reserved_bytes_peak_delta": 14187233280, + "segment_count_current": 692, + "segment_count_current_delta": 61, + "segment_count_peak": 692, + "segment_count_peak_delta": 61, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260921T075954665701Z.pickle", + "timestamp_utc": "2026-09-21T07:59:54.678572Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260921T075954665701Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260921T075954665701Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e278d015de85f4c84a1f43c90bf65bf305d0a6b2 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260921T075954665701Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbe3b123cd28a0fa8690262896616b94f850a949be321ca04d186dc784f050c7 +size 631482 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260921T075955731850Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260921T075955731850Z.json new file mode 100644 index 0000000000000000000000000000000000000000..778651893e9198d806f18c28c5f9477480f93a3d --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260921T075955731850Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624747520, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68340472832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51624747520, + "active_bytes_peak": 68340472832, + "allocated_bytes_current": 51624747520, + "allocated_bytes_peak": 68340472832, + "device": 6, + "inactive_split_bytes_current": 753720832, + "inactive_split_bytes_peak": 4477383168, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 71443677184, + "reserved_bytes_peak": 71443677184, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:55.731375Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624747520, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68340472832, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51624747520, + "active_bytes_peak": 68340472832, + "allocated_bytes_current": 51624747520, + "allocated_bytes_peak": 68340472832, + "device": 6, + "inactive_split_bytes_current": 753720832, + "inactive_split_bytes_peak": 4352139264, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 71443677184, + "reserved_bytes_peak": 71443677184, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:54.680411Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 753720832, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4477383168, + "inactive_split_bytes_peak_delta": 125243904, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 71443677184, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71443677184, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 692, + "segment_count_current_delta": 0, + "segment_count_peak": 692, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260921T075955731850Z.pickle", + "timestamp_utc": "2026-09-21T07:59:55.743711Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260921T075955731850Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260921T075955731850Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf1cde866c516d40c7b1982a191adecd8e44c1e2 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260921T075955731850Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:526b631e66e5de2f3705b21ca34a2eced7cd40e7449faddff456fd868d79ca6b +size 625894 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260921T075955883684Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260921T075955883684Z.json new file mode 100644 index 0000000000000000000000000000000000000000..403ca5c6e5b8c0a6b6c291b47fe2c4d63e9f0ca6 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260921T075955883684Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624747520, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68340472832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51624747520, + "active_bytes_peak": 68340472832, + "allocated_bytes_current": 51624747520, + "allocated_bytes_peak": 68340472832, + "device": 6, + "inactive_split_bytes_current": 753720832, + "inactive_split_bytes_peak": 4477383168, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 71443677184, + "reserved_bytes_peak": 71443677184, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:55.883244Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624747520, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68340472832, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51624747520, + "active_bytes_peak": 68340472832, + "allocated_bytes_current": 51624747520, + "allocated_bytes_peak": 68340472832, + "device": 6, + "inactive_split_bytes_current": 753720832, + "inactive_split_bytes_peak": 4477383168, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 71443677184, + "reserved_bytes_peak": 71443677184, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:55.745452Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 753720832, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4477383168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 71443677184, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71443677184, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 692, + "segment_count_current_delta": 0, + "segment_count_peak": 692, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260921T075955883684Z.pickle", + "timestamp_utc": "2026-09-21T07:59:55.894608Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260921T075955883684Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260921T075955883684Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f0c2dba6563ae3568bb404cac72e8d5390c10a1d --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260921T075955883684Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f9fe0221f3427006c104c02d52469ee0c246d73ae6cfcfae1017a1601d321e8 +size 625831 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260921T075956036529Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260921T075956036529Z.json new file mode 100644 index 0000000000000000000000000000000000000000..99cfa5d1733b4783b4b4083352494cf553f56d67 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260921T075956036529Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51624747520, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68340472832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51624747520, + "active_bytes_peak": 68340472832, + "allocated_bytes_current": 51624747520, + "allocated_bytes_peak": 68340472832, + "device": 6, + "inactive_split_bytes_current": 753720832, + "inactive_split_bytes_peak": 4477383168, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 71443677184, + "reserved_bytes_peak": 71443677184, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:56.036079Z", + "world_size": 8 + }, + "allocated_bytes_current": 51624747520, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68340472832, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51624747520, + "active_bytes_peak": 68340472832, + "allocated_bytes_current": 51624747520, + "allocated_bytes_peak": 68340472832, + "device": 6, + "inactive_split_bytes_current": 753720832, + "inactive_split_bytes_peak": 4477383168, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 71443677184, + "reserved_bytes_peak": 71443677184, + "segment_count_current": 692, + "segment_count_peak": 692, + "timestamp_utc": "2026-09-21T07:59:55.896243Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 753720832, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4477383168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 71443677184, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71443677184, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 692, + "segment_count_current_delta": 0, + "segment_count_peak": 692, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260921T075956036529Z.pickle", + "timestamp_utc": "2026-09-21T07:59:56.047679Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260921T075956036529Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260921T075956036529Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5222a423af60209f469992358c881852c3b8bf39 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260921T075956036529Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a6f2d0e97ffccae769703a159d4c0cb0faea933ed7712d124530b16e34b0c75 +size 625894 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260921T075953364955Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260921T075953364955Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0d9e61a3a51530d4c72b290be3dce81fb8697ddb --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260921T075953364955Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51625247744, + "active_bytes_current_delta": 21597801472, + "active_bytes_peak": 51625250816, + "active_bytes_peak_delta": 21539084288, + "after": { + "active_bytes_current": 51625247744, + "active_bytes_peak": 51625250816, + "allocated_bytes_current": 51625247744, + "allocated_bytes_peak": 51625250816, + "device": 7, + "inactive_split_bytes_current": 730151936, + "inactive_split_bytes_peak": 2917164032, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 57277415424, + "reserved_bytes_peak": 57277415424, + "segment_count_current": 632, + "segment_count_peak": 632, + "timestamp_utc": "2026-09-21T07:59:53.364395Z", + "world_size": 8 + }, + "allocated_bytes_current": 51625247744, + "allocated_bytes_current_delta": 21597801472, + "allocated_bytes_peak": 51625250816, + "allocated_bytes_peak_delta": 21539084288, + "before": { + "active_bytes_current": 30027446272, + "active_bytes_peak": 30086166528, + "allocated_bytes_current": 30027446272, + "allocated_bytes_peak": 30086166528, + "device": 7, + "inactive_split_bytes_current": 175736832, + "inactive_split_bytes_peak": 762939392, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 35888562176, + "reserved_bytes_peak": 35888562176, + "segment_count_current": 243, + "segment_count_peak": 243, + "timestamp_utc": "2026-09-21T07:59:22.956198Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 730151936, + "inactive_split_bytes_current_delta": 554415104, + "inactive_split_bytes_peak": 2917164032, + "inactive_split_bytes_peak_delta": 2154224640, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 57277415424, + "reserved_bytes_current_delta": 21388853248, + "reserved_bytes_peak": 57277415424, + "reserved_bytes_peak_delta": 21388853248, + "segment_count_current": 632, + "segment_count_current_delta": 389, + "segment_count_peak": 632, + "segment_count_peak_delta": 389, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260921T075953364955Z.pickle", + "timestamp_utc": "2026-09-21T07:59:53.378921Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260921T075953364955Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260921T075953364955Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7bdbc31d0337521da6ce7634031c78f35da07edf --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260921T075953364955Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85a8fbb6e4c12046fd02a1bcf9b5987025b69ced2e5d750ec85700d9fec5c5be +size 744681 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260921T075954665410Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260921T075954665410Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7aaca59180a61296568939123ce5861b13cb9ae3 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260921T075954665410Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51625247744, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68339725824, + "active_bytes_peak_delta": 16714475008, + "after": { + "active_bytes_current": 51625247744, + "active_bytes_peak": 68339725824, + "allocated_bytes_current": 51625247744, + "allocated_bytes_peak": 68339725824, + "device": 7, + "inactive_split_bytes_current": 730151936, + "inactive_split_bytes_peak": 4300218880, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 71431094272, + "reserved_bytes_peak": 71431094272, + "segment_count_current": 690, + "segment_count_peak": 690, + "timestamp_utc": "2026-09-21T07:59:54.665009Z", + "world_size": 8 + }, + "allocated_bytes_current": 51625247744, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68339725824, + "allocated_bytes_peak_delta": 16714475008, + "before": { + "active_bytes_current": 51625247744, + "active_bytes_peak": 51625250816, + "allocated_bytes_current": 51625247744, + "allocated_bytes_peak": 51625250816, + "device": 7, + "inactive_split_bytes_current": 730151936, + "inactive_split_bytes_peak": 2917164032, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 57277415424, + "reserved_bytes_peak": 57277415424, + "segment_count_current": 632, + "segment_count_peak": 632, + "timestamp_utc": "2026-09-21T07:59:53.380971Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 730151936, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4300218880, + "inactive_split_bytes_peak_delta": 1383054848, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 71431094272, + "reserved_bytes_current_delta": 14153678848, + "reserved_bytes_peak": 71431094272, + "reserved_bytes_peak_delta": 14153678848, + "segment_count_current": 690, + "segment_count_current_delta": 58, + "segment_count_peak": 690, + "segment_count_peak_delta": 58, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260921T075954665410Z.pickle", + "timestamp_utc": "2026-09-21T07:59:54.677641Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260921T075954665410Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260921T075954665410Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..568b0b9735878c7cfa9c1248726be0050b1c0b20 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260921T075954665410Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c28bebeb9d5ccb8afb21a037f070b8beae615d2ee492a4d9b42d2becc8b9361 +size 630695 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260921T075955731660Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260921T075955731660Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c16a47f492efaab10095c7e4ef97f289e9760011 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260921T075955731660Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51625247744, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68340393472, + "active_bytes_peak_delta": 667648, + "after": { + "active_bytes_current": 51625247744, + "active_bytes_peak": 68340393472, + "allocated_bytes_current": 51625247744, + "allocated_bytes_peak": 68340393472, + "device": 7, + "inactive_split_bytes_current": 730151936, + "inactive_split_bytes_peak": 4361417216, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 71431094272, + "reserved_bytes_peak": 71431094272, + "segment_count_current": 690, + "segment_count_peak": 690, + "timestamp_utc": "2026-09-21T07:59:55.731246Z", + "world_size": 8 + }, + "allocated_bytes_current": 51625247744, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68340393472, + "allocated_bytes_peak_delta": 667648, + "before": { + "active_bytes_current": 51625247744, + "active_bytes_peak": 68339725824, + "allocated_bytes_current": 51625247744, + "allocated_bytes_peak": 68339725824, + "device": 7, + "inactive_split_bytes_current": 730151936, + "inactive_split_bytes_peak": 4300218880, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 71431094272, + "reserved_bytes_peak": 71431094272, + "segment_count_current": 690, + "segment_count_peak": 690, + "timestamp_utc": "2026-09-21T07:59:54.679418Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 730151936, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4361417216, + "inactive_split_bytes_peak_delta": 61198336, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 71431094272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71431094272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 690, + "segment_count_current_delta": 0, + "segment_count_peak": 690, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260921T075955731660Z.pickle", + "timestamp_utc": "2026-09-21T07:59:55.742517Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260921T075955731660Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260921T075955731660Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..41dde99dc0b47c1611ebf19de400548fbda16c0d --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260921T075955731660Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3ac9d7426d1e7351e5ab6afa9a5122360399af368acd199543400102a4b2696 +size 624606 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260921T075955883521Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260921T075955883521Z.json new file mode 100644 index 0000000000000000000000000000000000000000..63ec69622902ec0553723c63707df0c18170a8f2 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260921T075955883521Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51625247744, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68340393472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51625247744, + "active_bytes_peak": 68340393472, + "allocated_bytes_current": 51625247744, + "allocated_bytes_peak": 68340393472, + "device": 7, + "inactive_split_bytes_current": 730151936, + "inactive_split_bytes_peak": 4420542976, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 71431094272, + "reserved_bytes_peak": 71431094272, + "segment_count_current": 690, + "segment_count_peak": 690, + "timestamp_utc": "2026-09-21T07:59:55.883123Z", + "world_size": 8 + }, + "allocated_bytes_current": 51625247744, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68340393472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51625247744, + "active_bytes_peak": 68340393472, + "allocated_bytes_current": 51625247744, + "allocated_bytes_peak": 68340393472, + "device": 7, + "inactive_split_bytes_current": 730151936, + "inactive_split_bytes_peak": 4361417216, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 71431094272, + "reserved_bytes_peak": 71431094272, + "segment_count_current": 690, + "segment_count_peak": 690, + "timestamp_utc": "2026-09-21T07:59:55.744153Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 730151936, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4420542976, + "inactive_split_bytes_peak_delta": 59125760, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 71431094272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71431094272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 690, + "segment_count_current_delta": 0, + "segment_count_peak": 690, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260921T075955883521Z.pickle", + "timestamp_utc": "2026-09-21T07:59:55.893856Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260921T075955883521Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260921T075955883521Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8b692831a0eb1fca643b509151dda51249129f6c --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260921T075955883521Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28356d8f5bacf1104ab5577c7c3ebda3a041f4bce9061b9d3d444c1866dea9ee +size 624574 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260921T075956036333Z.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260921T075956036333Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ce5d0e7ace9511de083b12bef18cfdc6d8796b65 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260921T075956036333Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 51625247744, + "active_bytes_current_delta": 0, + "active_bytes_peak": 68340393472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 51625247744, + "active_bytes_peak": 68340393472, + "allocated_bytes_current": 51625247744, + "allocated_bytes_peak": 68340393472, + "device": 7, + "inactive_split_bytes_current": 730151936, + "inactive_split_bytes_peak": 4420542976, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 71431094272, + "reserved_bytes_peak": 71431094272, + "segment_count_current": 690, + "segment_count_peak": 690, + "timestamp_utc": "2026-09-21T07:59:56.035940Z", + "world_size": 8 + }, + "allocated_bytes_current": 51625247744, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 68340393472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 51625247744, + "active_bytes_peak": 68340393472, + "allocated_bytes_current": 51625247744, + "allocated_bytes_peak": 68340393472, + "device": 7, + "inactive_split_bytes_current": 730151936, + "inactive_split_bytes_peak": 4420542976, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 71431094272, + "reserved_bytes_peak": 71431094272, + "segment_count_current": 690, + "segment_count_peak": 690, + "timestamp_utc": "2026-09-21T07:59:55.895384Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 730151936, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4420542976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 71431094272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 71431094272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 690, + "segment_count_current_delta": 0, + "segment_count_peak": 690, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260921T075956036333Z.pickle", + "timestamp_utc": "2026-09-21T07:59:56.046800Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260921T075956036333Z.pickle b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260921T075956036333Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d99304cc6622682133272507d5a24ca9b0378fb8 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260921T075956036333Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d520096f3d408fa6e4530c8cb2177d902f6eda21632027f49f8b707a0e8c424 +size 624606 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/run.env b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..cfd052cc2e4698d408b1abfb64ab0e39cd029e0e --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/run.env @@ -0,0 +1,102 @@ +backend=ultraep +config=KIMI_K2_L2 +family=tierB +source_model=Kimi-K2 +standard_num_layers=61 +base_rung= +repo=/workspace/Megatron-UltraEP +branch=ultraep-better-memory-dump-support +commit=c3a8a8a7 +num_layers=2 +hidden=7168 +ffn_hidden=18432 +moe_ffn_hidden=2048 +num_experts=384 +topk=8 +experts_per_rank=48 +expert_params=44040192 +heads=64 +kv_groups=64 +kv_channels=64 +qk_layernorm=1 +shared_ffn=2048 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=1 +q_lora_rank=1536 +kv_lora_rank=512 +qk_head_dim=128 +qk_pos_emb_head_dim=64 +v_head_dim=128 +dense_first_layers=1 +moe_layers=1 +rotary_base=50000 +rotary_percent=1.0 +rotary_scaling=32 +mscale=1.0 +mscale_all_dim=1.0 +qkv_bias=0 +router_balancing=seq_aux_loss +aux_loss_coeff=0 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +router_groups=1 +router_group_topk=1 +router_score_function=sigmoid +router_expert_bias=1 +router_bias_update_rate=1e-3 +router_topk_scaling_factor=2.827 +shared_expert_gate=0 +apply_layernorm_1p=0 +attention_output_gate=0 +linear_attention_type= +linear_attention_freq= +linear_conv_kernel_dim= +linear_key_head_dim= +linear_value_head_dim= +linear_num_key_heads= +linear_num_value_heads= +mtp_num_layers=0 +mtp_loss_scaling_factor=0.1 +no_weight_decay_cond_type= +max_position_embeddings=4096 +init_method_std=0.02 +device_initiated_grouped_gemm=0 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=682 +slot_policy=matched +slots_per_rank=48 +capacity_mode=dynamic +capacity=dynamic +cuda_graph=0 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=5 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=85 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/train.log b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..6a7011446cd81acf1ec78d62bc2faf0f4a40f25e --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/train.log @@ -0,0 +1,1104 @@ +[07:58:54] WARN: ultraep-better-memory-dump-support lacks --moe-router-fusion; KIMI_K2_L2 runs unfused (timing only, routing identical) +============================================================== + KIMI_K2_L2 / ultraep (slots/rank=48 policy=matched) + E=384 K=8 H'=2048 m=48 + L=2 seq=4096 gbs=8 (1 microbatches) + cap_mode=dynamic cap=dynamic cg=0 recompute=none + memory_dump=iteration snapshots=/workspace/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ c3a8a8a7 +============================================================== +W0921 07:58:56.222000 560074 torch/distributed/run.py:766] +W0921 07:58:56.222000 560074 torch/distributed/run.py:766] ***************************************** +W0921 07:58:56.222000 560074 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0921 07:58:56.222000 560074 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 2 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 18432 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 7168 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 4096 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_capacity_monitor ............................ False + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 2048 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1] + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 48 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... True + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... 1 + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... 1 + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... sigmoid + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. 2.827 + moe_shared_expert_intermediate_size ............. 2048 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... False + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 1.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 64 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 384 + num_layers ...................................... 2 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... 1536 + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 50000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 32.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 5 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 1 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 14357 is connected to is connected to is connected to is connected to is connected to 77777 peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77777 + + + + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank3]:[W921 07:59:13.367885734 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W921 07:59:13.367945286 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W921 07:59:13.367966287 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W921 07:59:13.368032635 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W921 07:59:13.368065994 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W921 07:59:13.368072289 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W921 07:59:13.369362208 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.036 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W921 07:59:13.532529577 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.664 seconds +time to initialize megatron (seconds): 7.256 +[after megatron is initialized] datetime: 2026-09-21 07:59:19 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 5594455040 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 9 +Params for bucket 1 (360579072 elements, 360579072 padded size): + module.output_layer.weight +Params for bucket 2 (44047360 elements, 44047360 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (45163520 elements, 45163520 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_q_up_proj.weight + module.decoder.layers.1.self_attention.linear_q_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight + module.decoder.layers.1.self_attention.linear_q_down_proj.weight +Params for bucket 4 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight +Params for bucket 5 (132127744 elements, 132127744 padded size): + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc2.weight +Params for bucket 6 (264241152 elements, 264241152 padded size): + module.decoder.layers.0.mlp.linear_fc1.weight +Params for bucket 7 (42411008 elements, 42411008 padded size): + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.self_attention.linear_q_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_q_up_proj.weight + module.decoder.layers.0.self_attention.linear_q_down_proj.weight +Params for bucket 8 (58720256 elements, 58720256 padded size): + module.decoder.layers.0.self_attention.linear_proj.weight +Params for bucket 9 (360586240 elements, 360586240 padded size): + module.decoder.layers.0.input_layernorm.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight47 + module.decoder.layers.1.mlp.experts.linear_fc2.weight45 + module.decoder.layers.1.mlp.experts.linear_fc2.weight46 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight44 + module.decoder.layers.1.mlp.experts.linear_fc2.weight42 + module.decoder.layers.1.mlp.experts.linear_fc2.weight43 +Params for bucket 3 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight41 + module.decoder.layers.1.mlp.experts.linear_fc2.weight39 + module.decoder.layers.1.mlp.experts.linear_fc2.weight40 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight38 + module.decoder.layers.1.mlp.experts.linear_fc2.weight36 + module.decoder.layers.1.mlp.experts.linear_fc2.weight37 +Params for bucket 5 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight35 + module.decoder.layers.1.mlp.experts.linear_fc2.weight33 + module.decoder.layers.1.mlp.experts.linear_fc2.weight34 +Params for bucket 6 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight32 + module.decoder.layers.1.mlp.experts.linear_fc2.weight30 + module.decoder.layers.1.mlp.experts.linear_fc2.weight31 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight29 + module.decoder.layers.1.mlp.experts.linear_fc2.weight28 + module.decoder.layers.1.mlp.experts.linear_fc2.weight27 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight26 + module.decoder.layers.1.mlp.experts.linear_fc2.weight25 + module.decoder.layers.1.mlp.experts.linear_fc2.weight24 +Params for bucket 9 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight23 + module.decoder.layers.1.mlp.experts.linear_fc2.weight22 + module.decoder.layers.1.mlp.experts.linear_fc2.weight21 +Params for bucket 10 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight20 + module.decoder.layers.1.mlp.experts.linear_fc2.weight19 + module.decoder.layers.1.mlp.experts.linear_fc2.weight18 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight17 + module.decoder.layers.1.mlp.experts.linear_fc2.weight16 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 +Params for bucket 17 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight47 + module.decoder.layers.1.mlp.experts.linear_fc1.weight46 +Params for bucket 18 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight45 + module.decoder.layers.1.mlp.experts.linear_fc1.weight44 +Params for bucket 19 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight43 + module.decoder.layers.1.mlp.experts.linear_fc1.weight42 +Params for bucket 20 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight41 + module.decoder.layers.1.mlp.experts.linear_fc1.weight40 +Params for bucket 21 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight39 + module.decoder.layers.1.mlp.experts.linear_fc1.weight38 +Params for bucket 22 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight37 + module.decoder.layers.1.mlp.experts.linear_fc1.weight36 +Params for bucket 23 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight35 + module.decoder.layers.1.mlp.experts.linear_fc1.weight34 +Params for bucket 24 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight33 + module.decoder.layers.1.mlp.experts.linear_fc1.weight32 +Params for bucket 25 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight31 + module.decoder.layers.1.mlp.experts.linear_fc1.weight30 +Params for bucket 26 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight29 + module.decoder.layers.1.mlp.experts.linear_fc1.weight28 +Params for bucket 27 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight27 + module.decoder.layers.1.mlp.experts.linear_fc1.weight26 +Params for bucket 28 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight25 + module.decoder.layers.1.mlp.experts.linear_fc1.weight24 +Params for bucket 29 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight23 + module.decoder.layers.1.mlp.experts.linear_fc1.weight22 +Params for bucket 30 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight21 + module.decoder.layers.1.mlp.experts.linear_fc1.weight20 +Params for bucket 31 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight19 + module.decoder.layers.1.mlp.experts.linear_fc1.weight18 +Params for bucket 32 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight17 + module.decoder.layers.1.mlp.experts.linear_fc1.weight16 +Params for bucket 33 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 +Params for bucket 34 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 +Params for bucket 35 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 +Params for bucket 36 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 +Params for bucket 37 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 +Params for bucket 38 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 39 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 +Params for bucket 40 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-21 07:59:22 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 40 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(40, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 0152a0378ab8ae1561633a6718ca974c-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 0152a0378ab8ae1561633a6718ca974c-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 0152a0378ab8ae1561633a6718ca974c-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-09-21 07:59:22 +done with setup ... +training ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3535.90, 3538.86) + train/valid/test-data-iterators-setup ..........: (96.31, 112.64) +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-09-21 07:59:22 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-09-21 07:59:53] iteration 1/ 5 | consumed samples: 8 | elapsed time per iteration (ms): 30406.8 | throughput per GPU (TFLOP/s/GPU): 1.2 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.227924E+01 | loss scale: 1.0 | grad norm: 12.617 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 17.55 +Number of parameters in embedding layers in billions: 0.72 +Total number of parameters in billions: 18.28 +Number of parameters in most loaded shard in billions: 18.2753 +Theoretical memory footprints: weight and optimizer=130715.23 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 49232.8642578125 | max allocated: 49232.86474609375 | reserved: 54586.0 | max reserved: 54586.0 +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-09-21 07:59:54] iteration 2/ 5 | consumed samples: 16 | elapsed time per iteration (ms): 1301.3 | throughput per GPU (TFLOP/s/GPU): 27.2 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.227088E+01 | loss scale: 1.0 | grad norm: 12.533 | number of skipped iterations: 0 | number of nan iterations: 0 | +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-09-21 07:59:55] iteration 3/ 5 | consumed samples: 24 | elapsed time per iteration (ms): 1066.2 | throughput per GPU (TFLOP/s/GPU): 33.2 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.088442E+01 | loss scale: 1.0 | grad norm: 11.038 | number of skipped iterations: 0 | number of nan iterations: 0 | +/workspace/Megatron-UltraEP/megatron/core/distributed/param_and_grad_buffer.py:282: UserWarning: The next bucket's parameter all-gather operation has already been dispatched. This may be caused by a mismatch between the order of parameter registration and forward pass execution, which will hurt the communication-computation overlap performance. + warnings.warn( +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-09-21 07:59:55] iteration 4/ 5 | consumed samples: 32 | elapsed time per iteration (ms): 151.8 | throughput per GPU (TFLOP/s/GPU): 233.1 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.447834E+01 | loss scale: 1.0 | grad norm: 72.934 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-09-21 07:59:56] iteration 5/ 5 | consumed samples: 40 | elapsed time per iteration (ms): 152.8 | throughput per GPU (TFLOP/s/GPU): 231.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.809694E+01 | loss scale: 1.0 | grad norm: 108.410 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-09-21 07:59:56 +[rank0]:[W921 07:59:57.268924475 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W921 07:59:58.395854026 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W921 07:59:58.396595762 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W921 07:59:58.398809146 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W921 07:59:58.781327517 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W921 07:59:58.800406553 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W921 07:59:58.801288846 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W921 07:59:58.852768017 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0921 08:00:04.898000 560074 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0921 08:00:04.898000 560074 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 560366 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 560074 got signal: 15 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/memory_summary.json b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/memory_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..cb90042f9b537db7485a35c72722a87563494079 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/memory_summary.json @@ -0,0 +1,66 @@ +{ + "runs": [ + { + "run_dir": "KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1", + "config": "KIMI_K2_L2", + "backend": "ultraep", + "num_layers": "2", + "moe_layers": "1", + "seq_length": "4096", + "global_batch_size": "8", + "cuda_graph": "1", + "capacity_mode": "static", + "capacity": "1.17578125", + "memory_dump_mode": "iteration", + "branch": "ultraep-better-memory-dump-support", + "commit": "c3a8a8a7", + "status": "OK", + "max_allocated_gib": 47.984477043151855, + "max_reserved_gib": 53.119140625, + "fragmentation_gib": 5.1346635818481445, + "theoretical_weight_optim_gib": 127.651591796875, + "peak_sampled_gib": 88.2568359375, + "snapshot_pickle_count": 48, + "snapshot_json_count": 48, + "snapshot_expected_each": 48, + "snapshot_complete": 1, + "effective_peak_gib": 88.2568359375 + }, + { + "run_dir": "KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1", + "config": "KIMI_K2_L2", + "backend": "ultraep", + "num_layers": "2", + "moe_layers": "1", + "seq_length": "4096", + "global_batch_size": "8", + "cuda_graph": "0", + "capacity_mode": "dynamic", + "capacity": "dynamic", + "memory_dump_mode": "iteration", + "branch": "ultraep-better-memory-dump-support", + "commit": "c3a8a8a7", + "status": "OK", + "max_allocated_gib": 48.07896947860718, + "max_reserved_gib": 53.306640625, + "fragmentation_gib": 5.227671146392822, + "theoretical_weight_optim_gib": 127.651591796875, + "peak_sampled_gib": 87.6240234375, + "snapshot_pickle_count": 40, + "snapshot_json_count": 40, + "snapshot_expected_each": 40, + "snapshot_complete": 1, + "effective_peak_gib": 87.6240234375 + } + ], + "comparisons": [ + { + "config": "KIMI_K2_L2", + "backend": "ultraep", + "dynamic_nocg_status": "OK", + "dynamic_nocg_peak_gib": 87.6240234375, + "cap_1p17578125_status": "OK", + "cap_1p17578125_peak_gib": 88.2568359375 + } + ] +} \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/memory_summary.tsv b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/memory_summary.tsv new file mode 100644 index 0000000000000000000000000000000000000000..db172b286fa69c1dc66fc10b732b7f3b26c1aa4d --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/memory_summary.tsv @@ -0,0 +1,3 @@ +run_dir config backend cuda_graph capacity_mode capacity status effective_peak_gib max_allocated_gib max_reserved_gib fragmentation_gib peak_sampled_gib snapshot_pickle_count snapshot_json_count snapshot_expected_each snapshot_complete branch commit +KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1 KIMI_K2_L2 ultraep 1 static 1.17578125 OK 88.2568359375 47.984477043151855 53.119140625 5.1346635818481445 88.2568359375 48 48 48 1 ultraep-better-memory-dump-support c3a8a8a7 +KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1 KIMI_K2_L2 ultraep 0 dynamic dynamic OK 87.6240234375 48.07896947860718 53.306640625 5.227671146392822 87.6240234375 40 40 40 1 ultraep-better-memory-dump-support c3a8a8a7 diff --git a/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/results.tsv b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/results.tsv new file mode 100644 index 0000000000000000000000000000000000000000..4195beb370b8cdec33612ca853cc25a0dd7f9267 --- /dev/null +++ b/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/results.tsv @@ -0,0 +1,3 @@ +config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir +KIMI_K2_L2 ultraep matched 48 4096 8 1.17578125 none 1 OK 118.50 118.50 90375 /workspace/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__cap1p17578125__memiteration__it6/rep1 +KIMI_K2_L2 ultraep matched 48 4096 8 dynamic none 1 OK NA NA 89727 /workspace/granularity_next_b200/tierb_models/memory/KIMI_K2_L2/ultraep/tierbmem/tierbmem_gbs8/KIMI_K2_L2__ultraep__matched__seq4096__gbs8__capdynamic__nocg__memiteration__it5/rep1