diff --git a/moonep/tierbmem/manifest.txt b/moonep/tierbmem/manifest.txt new file mode 100644 index 0000000000000000000000000000000000000000..3e8332f042dd615f1028a64c5a6d20b629ba6095 --- /dev/null +++ b/moonep/tierbmem/manifest.txt @@ -0,0 +1,19 @@ +timestamp: 2026-08-26T19:29:51+00:00 +host: sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 +gpu: NVIDIA B200 x8 +driver: 580.95.05 +study_scripts_branch: granularity-moonep +study_scripts_commit: ef3305e +campaign: moonep +backend: moonep +phase: tierbmem +gbs: 8 +seq: 4096 +configs: G1 G2 G3 G4 +capacity: 1.25 +repeats: 3 +nsys: 1 +megatron_repo: /workspace/Megatron-MoE-Echo +megatron_branch_required: moonep_backend_better_memory_dump_support +megatron_branch_actual: moonep_backend_better_memory_dump_support +megatron_commit: 453b84c7 diff --git a/moonep/tierbmem/pip_freeze.txt b/moonep/tierbmem/pip_freeze.txt new file mode 100644 index 0000000000000000000000000000000000000000..e0d487187d80c876abad896ab0c1743317a451a9 --- /dev/null +++ b/moonep/tierbmem/pip_freeze.txt @@ -0,0 +1,317 @@ +absl-py==2.3.0 +aiohappyeyeballs @ file:///rapids/aiohappyeyeballs-2.6.1-py3-none-any.whl#sha256=f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 +aiohttp @ file:///rapids/aiohttp-3.12.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=94f98e0e5a49f89b252e115844f756c04fc8050f38252a32a3dd994ce8121f10 +aiosignal @ file:///rapids/aiosignal-1.3.2-py2.py3-none-any.whl#sha256=45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5 +annotated-types==0.7.0 +anyio==4.9.0 +apex @ file:///opt/pytorch/apex/dist/apex-0.1-cp312-cp312-linux_x86_64.whl#sha256=a1508e4a386362e9c8f46c92ccada9110e421212b964c694235d0fd5a224b170 +argon2-cffi==25.1.0 +argon2-cffi-bindings==21.2.0 +arrow==1.3.0 +asciitree @ file:///rapids/asciitree-0.3.3-py3-none-any.whl#sha256=3e507ccfaeb417823158091fdb9d26b0205f6a59ba680b479a71db65c73d01d4 +asttokens==3.0.0 +astunparse==1.6.3 +async-lru==2.0.5 +attrs==25.3.0 +audioread==3.0.1 +babel==2.17.0 +beautifulsoup4==4.13.4 +black==25.1.0 +bleach==6.2.0 +blis==0.7.11 +cachetools @ file:///rapids/cachetools-6.0.0-py3-none-any.whl#sha256=82e73ba88f7b30228b5507dce1a1f878498fc669d972aef2dde4f3a3c24f103e +catalogue==2.0.10 +certifi==2025.4.26 +cffi==1.17.1 +charset-normalizer==3.4.2 +click==8.5.0 +cloudpathlib==0.21.1 +cloudpickle @ file:///rapids/cloudpickle-3.1.1-py3-none-any.whl#sha256=c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e +cmake==3.31.6 +comm==0.2.2 +confection==0.1.5 +contourpy==1.3.2 +cuda-bindings @ file:///rapids/cuda_bindings-12.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=ff0e28d1e34758654b9c961e1f55e4786e49aee6a4dbceaf3cc24c46c672df7e +cuda-python @ file:///rapids/cuda_python-12.9.0-py3-none-any.whl#sha256=926acba49b2c0a0374c61b7c98f337c085199cf51cdfe4d6423c4129c20547a7 +cudf @ file:///rapids/cudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b779a8e8c0e8577b9b560895958496da211bfac49cdc4f11a49e8d18cb82e348 +cudf-polars @ file:///rapids/cudf_polars-25.4.0-py3-none-any.whl#sha256=227b6c5147e93ea86282c8f7219118f916769fa2ca0cdd931e97a148aa66b869 +cugraph @ file:///rapids/cugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b837c2dc2f7b235c702e4f8df64479b8fcbb342f848516d5faf0cdc4b5f31f29 +cugraph-service-client @ file:///rapids/cugraph_service_client-25.4.0-py3-none-any.whl#sha256=7e568c3e41154b9752f5173555ffcc282ac52d72343feeb79653b2e2b1232470 +cugraph-service-server @ file:///rapids/cugraph_service_server-25.4.0-py3-none-any.whl#sha256=a2830dcffafcccb16e12ebf85b98f8f247869d38c01ab4dd8664a1e7439d31be +cuml @ file:///rapids/cuml-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1e90a464a18b094e24797f1676688c5d7696c4cf97c934504214a53d5aa09aa5 +cupy-cuda12x @ file:///rapids/cupy_cuda12x-13.3.0-cp312-cp312-linux_x86_64.whl#sha256=6a2dfa0aa1e8189a10d1534276c08f1d2f73b747fd28bfbe7b9f4ef66d9af73f +cuvs @ file:///rapids/cuvs-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b12f6b9f80e053a3b38542c2947d2516e468c02cad54a13d8b1a4a61d5aa7914 +cycler==0.12.1 +cymem==2.0.11 +Cython==3.1.1 +dask @ file:///rapids/dask-2025.2.0-py3-none-any.whl#sha256=f0fdeef6ceb0a06569d456c9e704f220f7f54e80f3a6ea42ab98cea6bc642b6e +dask-cuda @ file:///rapids/dask_cuda-25.4.0-py3-none-any.whl#sha256=442d87da0a95d00ca2a262b64c76e785a32ad905c82000fa2227932f81390e19 +dask-cudf @ file:///rapids/dask_cudf-25.4.0-py3-none-any.whl#sha256=5e7cea0b26daea17da6e0bb457435e24460f76ba740358a3a18e239d6015c95c +datasets==5.0.1 +debugpy==1.8.14 +decorator==5.2.1 +-e git+https://github.com/deepseek-ai/DeepEP.git@34152ae28f80bcc3ee38d7a12cb2ad87cfd4ea72#egg=deep_ep +defusedxml==0.7.1 +dill==0.4.1 +distributed @ file:///rapids/distributed-2025.2.0-py3-none-any.whl#sha256=368462084be47092d0a835f6c64c9b852da64dc88302b76d5f93bd5131228c67 +distributed-ucxx @ file:///rapids/distributed_ucxx-0.43.0-py3-none-any.whl#sha256=94f60d0ccb0c94ba8dbfa1eaf455874d18aaab17e2ae218c38590e9fdc2d721d +dm-tree==0.1.9 +einops==0.8.1 +execnet==2.1.1 +executing==2.2.0 +expecttest==0.3.0 +fasteners @ file:///rapids/fasteners-0.19-py3-none-any.whl#sha256=758819cb5d94cdedf4e836988b74de396ceacb8e2794d21f82d131fd9ee77237 +fastjsonschema==2.21.1 +fastrlock @ file:///rapids/fastrlock-0.8.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl#sha256=5f13ec08f1adb1aa916c384b05ecb7dbebb8df9ea81abd045f60941c6283a670 +filelock==3.18.0 +flash_attn @ file:///opt/pytorch/flash_attn-2.7.4.post1-cp312-cp312-linux_x86_64.whl#sha256=a10455d6c56b387785b7413fd0bbcb392b42b3d4f4b88fda5d86cc8a9ce9182a +fonttools==4.58.1 +fqdn==1.5.1 +frozenlist @ file:///rapids/frozenlist-1.6.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=4e1be9111cb6756868ac242b3c2bd1f09d9aea09846e4f5c23715e7afb647103 +fsspec==2025.5.1 +gast==0.6.0 +grpcio==1.62.1 +h11==0.16.0 +hf-xet==1.6.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.28.0 +hypothesis==6.130.8 +idna==3.10 +importlib_metadata==8.7.0 +iniconfig==2.1.0 +intel-openmp==2021.4.0 +ipykernel==6.29.5 +ipython==9.3.0 +ipython_pygments_lexers==1.1.1 +isoduration==20.11.0 +isort==6.0.1 +jedi==0.19.2 +Jinja2==3.1.6 +joblib==1.5.1 +json5==0.12.0 +jsonpointer==3.0.0 +jsonschema==4.24.0 +jsonschema-specifications==2025.4.1 +jupyter-events==0.12.0 +jupyter-lsp==2.2.5 +jupyter_client==8.6.3 +jupyter_core==5.8.1 +jupyter_server==2.16.0 +jupyter_server_terminals==0.5.3 +jupyterlab==4.4.3 +jupyterlab_code_formatter==3.0.2 +jupyterlab_pygments==0.3.0 +jupyterlab_server==2.27.3 +jupyterlab_tensorboard_pro @ file:///builder/jupyterlab_tensorboard_pro-4.0.0-py2.py3-none-any.whl#sha256=6bfcb5ced3358eb72150f4b2ab91312faef2f0ca14d845e8379d8fa52df6902e +jupytext==1.17.2 +kiwisolver==1.4.8 +kvikio @ file:///rapids/kvikio-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=8b40fd8ea9e4a2974c04b50b7fc26b06094ac807f9a233281d0bafbaa71b1d85 +langcodes==3.5.0 +language_data==1.3.0 +lazy_loader==0.4 +libcudf @ file:///rapids/libcudf-25.4.0-py3-none-linux_x86_64.whl#sha256=238320b4958c1737bba2a609673e2539a78216b811b5498a68e00395d9b021a4 +libcugraph @ file:///rapids/libcugraph-25.4.0-py3-none-linux_x86_64.whl#sha256=29085e901b2f7569ff2653bb1355fe6fad20f2fcaa585f5649da9867d8df766d +libcuml @ file:///rapids/libcuml-25.4.0-py3-none-linux_x86_64.whl#sha256=8e2e444c72b0e995e15a48da37ff12fb410a47a2baa69ba86ccc9b1a3db91e8c +libcuvs @ file:///rapids/libcuvs-25.4.0-py3-none-linux_x86_64.whl#sha256=e2f8db61c92b769c590dd752f43439ddd62b2f6592a3c337362b6c1ef220c3b9 +libkvikio @ file:///rapids/libkvikio-25.4.0-py3-none-linux_x86_64.whl#sha256=da8a5713b05fc478bf40529a3e0fb61a2d605bb5a70ca4d461f96a6005dbe8b7 +libraft @ file:///rapids/libraft-25.4.0-py3-none-linux_x86_64.whl#sha256=b921dec005aa4a321dd156776a18376feb47c96a5b67eec125c8a8e19edd1e24 +librmm @ file:///rapids/librmm-25.4.0-py3-none-linux_x86_64.whl#sha256=ff951811bda729d4fa1109ab906225ae1f489a6ec2eeab5d1b674cc34bd4422f +librmm-cu12 @ file:///rapids/librmm_cu12-25.4.0-py3-none-any.whl#sha256=29296ef2b0e90679a0db280e86015be12d6ab9f4e62a181aa085f62dbadd06b3 +librosa==0.11.0 +libucxx @ file:///rapids/libucxx-0.43.0-py3-none-linux_x86_64.whl#sha256=f15849d7ac2fb68f0fd8998838a13321ddcfb0c594f7e6cbc401a3bcc4750566 +lightning-thunder @ file:///opt/pytorch/lightning-thunder +lightning-utilities==0.14.3 +lintrunner==0.12.7 +llvmlite==0.42.0 +locket @ file:///rapids/locket-1.0.0-py2.py3-none-any.whl#sha256=b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3 +looseversion==1.3.0 +marisa-trie==1.2.1 +Markdown==3.8 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +matplotlib==3.10.3 +matplotlib-inline==0.1.7 +mdit-py-plugins==0.4.2 +mdurl==0.1.2 +-e git+ssh://git@github.com/sriram-17-17/my-Megatron-MoE-Echo.git@453b84c7376f213d3e332c2f9b87b60bf96c4921#egg=megatron_core +mistune==3.1.3 +mkl==2021.1.1 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +ml_dtypes==0.6.0 +mock==5.2.0 +-e git+ssh://git@github.com/sriram-17-17/MoonEP.git@c27d35da15f25c112218795894504a65c16f4879#egg=moonep +mpmath==1.3.0 +msgpack==1.1.0 +multidict @ file:///rapids/multidict-6.4.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=a920f9cf2abdf6e493c519492d892c362007f113c94da4c239ae88429835bad1 +multiprocess==0.70.19 +murmurhash==1.0.13 +mypy_extensions==1.1.0 +nbclient==0.10.2 +nbconvert==7.16.6 +nbformat==5.10.4 +nest-asyncio==1.6.0 +networkx==3.5 +ninja==1.11.1.4 +notebook==7.4.3 +notebook_shim==0.2.4 +numba==0.59.1 +numba-cuda @ file:///rapids/numba_cuda-0.4.0-py3-none-any.whl#sha256=5b95a5f42068771e37142944d44fe844fa87f1b2f89051461a4cb0af91bcc28a +numcodecs @ file:///rapids/numcodecs-0.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=da2230484e6102e5fa3cc1a5dd37ca1f92dfbd183d91662074d6f7574e3e8f53 +numpy==1.26.4 +nvdlfw_inspect @ git+https://github.com/NVIDIA/nvidia-dlfw-inspect.git@6b60eb0e675606fb2fbbfcb12667ecbec75eaf87 +nvfuser==0.2.27a0+9bf5aca +nvidia-cuda-cccl-cu12==12.9.27 +nvidia-cudnn-frontend @ file:///opt/pytorch/lightning-thunder/tmp/cudnn_frontend +nvidia-cutlass-dsl==4.4.2 +nvidia-cutlass-dsl-libs-base==4.4.2 +nvidia-dali-cuda120==1.50.0 +nvidia-ml-py @ file:///rapids/nvidia_ml_py-12.575.51-py3-none-any.whl#sha256=eb8641800d98ce40a22f479873f34b482e214a7e80349c63be51c3919845446e +nvidia-modelopt==0.29.0 +nvidia-modelopt-core==0.29.0 +nvidia-nvcomp-cu12==4.2.0.14 +nvidia-nvimgcodec-cu12==0.5.0.13 +nvidia-nvjpeg-cu12==12.4.0.16 +nvidia-nvjpeg2k-cu12==0.8.1.40 +nvidia-nvshmem-cu12==3.7.2 +nvidia-nvtiff-cu12==0.5.0.67 +nvidia-resiliency-ext==0.4.0 +nvtx @ file:///rapids/nvtx-0.2.11-cp312-cp312-linux_x86_64.whl#sha256=73521e37cba42b4db0b3cd68fc5eb814e55c500a61eeed6edcdf6746315ac53e +nx-cugraph @ file:///rapids/nx_cugraph-25.4.0-py3-none-any.whl#sha256=e625444486cc899a77bd48e7da76a2edd95c0006dd95fbf323745feae7f33111 +onnx @ file:///opt/pytorch/pytorch/third_party/onnx +onnx-ir==1.0.0 +onnxscript==0.7.1 +opt_einsum==3.4.0 +optree==0.16.0 +overrides==7.7.0 +packaging==23.2 +pandas @ file:///rapids/pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319 +pandocfilters==1.5.1 +parso==0.8.4 +partd @ file:///rapids/partd-1.4.2-py3-none-any.whl#sha256=978e4ac767ec4ba5b86c6eaa52e5a2a3bc748a2ca839e8cc798f1cc6ce6efb0f +pathspec==0.12.1 +pexpect==4.9.0 +pillow==11.2.1 +platformdirs==4.3.8 +pluggy==1.6.0 +ply @ file:///rapids/ply-3.11-py2.py3-none-any.whl#sha256=096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce +polars @ file:///rapids/polars-1.25.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=f7fcbb4f476784384ccda48757fca4e8c2e2c5a0a3aef3717aaf56aee4e30e09 +polygraphy==0.49.20 +pooch==1.8.2 +preshed==3.0.10 +prometheus_client==0.22.1 +prompt_toolkit==3.0.51 +propcache @ file:///rapids/propcache-0.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=563f9d8c03ad645597b8d010ef4e9eab359faeb11a0a2ac9f7b4bc8c28ebef25 +protobuf==4.24.4 +psutil==7.0.0 +ptyprocess==0.7.0 +PuLP==3.2.1 +pure_eval==0.2.3 +pyarrow==25.0.1 +pybind11==2.13.6 +pybind11_global==2.13.6 +pycocotools @ git+https://github.com/nvidia/cocoapi.git@08350a157b41f0755a536441b5abec89ab934c5b#subdirectory=PythonAPI +pycparser==2.22 +pydantic==2.11.5 +pydantic_core==2.33.2 +Pygments==2.19.1 +pylibcudf @ file:///rapids/pylibcudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=401b8bfd022d06f84e120779ca7c3d01a28b68762aaa8f7873b10d6b1975cf6a +pylibcugraph @ file:///rapids/pylibcugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=061c7389a88c815b54fd5d3a3584fb87bf693e5602f5be335260d0ba575ba557 +pylibcugraphops @ file:///rapids/pylibcugraphops-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=a25ef4b22166c80718d5c7e5bddb755e81a2fe340fa5becd0ac36ec3edfb208a +pylibraft @ file:///rapids/pylibraft-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=f2ecd23fe51f9c2caa87bbbe85cb32fcaff4ecfb52ceb9a9991bc50f82782c2e +pylibwholegraph @ file:///rapids/pylibwholegraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b82913d1e38bda76d8c272fbd8cc9072d987209d3d47cc93a464a37c2925aca2 +pynvjitlink @ file:///rapids/pynvjitlink-0.3.0-cp312-cp312-linux_x86_64.whl#sha256=066caf1314730643d3b48c9847bd65789defdd6a7c78993176eb3d8bbe3f4919 +pynvml @ file:///rapids/pynvml-12.0.0-py3-none-any.whl#sha256=fdff84b62a27dbe98e08e1a647eb77342bef1aebe0878bcd15e99a83fcbecb9e +pyparsing==3.2.3 +pytest==8.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==15.1 +pytest-shard==0.1.2 +pytest-xdist==3.7.0 +python-dateutil==2.9.0.post0 +python-hostlist==2.2.1 +python-json-logger==3.3.0 +pytorch-triton @ file:///tmp/dist/pytorch_triton-3.3.0%2Bgit96316ce52.nvinternal-cp312-cp312-linux_x86_64.whl#sha256=6d6921772b9dfd0243c055e0d054de42a113e92cabcfdd1d3e80a5ad9c08cf7d +pytz @ file:///rapids/pytz-2023.4-py2.py3-none-any.whl#sha256=f90ef520d95e7c46951105338d918664ebfd6f1d995bd7d153127ce90efafa6a +PyYAML==6.0.2 +pyzmq==26.4.0 +raft-dask @ file:///rapids/raft_dask-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=36cf37ba6ce35454c4729c2cb597a360f22780eefb11d514a16a07137da5eb6c +rapids-dask-dependency @ file:///rapids/rapids_dask_dependency-25.4.0a0-py3-none-any.whl#sha256=484faca626e7760112a7d8ad3426f5bdd720b609006d8dca87c2607076a348bd +rapids-logger @ file:///rapids/rapids_logger-0.1.18-py3-none-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=3617b2d86291ac6cb957542895f20fc9737ace29773ca90337e6fb04c73bd120 +referencing==0.36.2 +regex==2024.11.6 +requests==2.32.3 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rich==14.0.0 +rmm @ file:///rapids/rmm-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1824bcab3836296666980a997ddcb9c4bca24dfb873012cfa0a7ecea583ca712 +rpds-py==0.25.1 +safetensors==0.5.3 +scikit-learn==1.6.1 +scipy==1.15.3 +Send2Trash==1.8.3 +setuptools==78.1.1 +shellingham==1.5.4 +six==1.16.0 +smart-open==7.1.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.13.1 +soupsieve==2.7 +soxr==0.5.0.post1 +spacy==3.7.5 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +srsly==2.5.1 +stack-data==0.6.3 +sympy==1.14.0 +tabulate==0.9.0 +tbb==2021.13.1 +tblib @ file:///rapids/tblib-3.1.0-py3-none-any.whl#sha256=670bb4582578134b3d81a84afa1b016128b429f3d48e6cbbaecc9d15675e984e +tensorboard==2.16.2 +tensorboard-data-server==0.7.2 +tensorrt @ file:///workspace/TensorRT-10.11.0.33/python/tensorrt-10.11.0.33-cp312-none-linux_x86_64.whl#sha256=c2f18f72758a3d54a71901495c8fda9e7094a8bcdd2a5b6b58967fbf05733b3a +terminado==0.18.1 +thinc==8.2.5 +threadpoolctl==3.6.0 +thriftpy2 @ file:///rapids/thriftpy2-0.5.2-cp312-cp312-linux_x86_64.whl#sha256=c0c7d939bdc5ccf7739154959938445d5fdf72f1d6e12bd563947a36f67e2147 +tinycss2==1.4.0 +toolz @ file:///rapids/toolz-1.0.0-py3-none-any.whl#sha256=292c8f1c4e7516bf9086f8850935c799a874039c8bcf959d47b600e4c44a6236 +torch @ file:///opt/transfer/torch-2.8.0a0%2B5228986c39.nv25.6-29946474-cp312-cp312-linux_x86_64.whl#sha256=7a3c9ff5288d8102a31296b3d62491f3f47c8820a5c9aaefc7dc0fe20e1e8ad5 +torch_tensorrt @ file:///opt/pytorch/torch_tensorrt/dist/torch_tensorrt-2.8.0a0-cp312-cp312-linux_x86_64.whl#sha256=26f4d7a04e7e6e017a5b917b8b98b83720bb2bb0195f48d25f52d567d286bfe1 +torchao @ file:///opt/pytorch/ao +torchprofile==0.0.4 +torchvision @ file:///opt/transfer/torchvision-0.22.0a0%2B95f10a4e-29946474-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=746a3c674cc61680a86f4a14b374b21af1b5c21f629c42765b40c9494f6b234d +tornado==6.5.1 +tqdm==4.67.1 +traitlets==5.14.3 +transformer_engine @ file:///pvcdata/aiinfra/b-aghosh/wheels/transformer_engine-2.10.0.dev0%2B10f6128d-cp312-cp312-linux_x86_64.whl#sha256=35cf5f4b83b2baad864e12e512bb422e0929409ac05610812fa521d4942d0189 +treelite @ file:///rapids/treelite-4.4.1-py3-none-linux_x86_64.whl#sha256=174741724205540a1a5b37da81483aeff55d89548f42b3461cf07acac7ae96eb +typer==0.16.0 +types-dataclasses==0.6.6 +types-python-dateutil==2.9.0.20250516 +typing-inspection==0.4.1 +typing_extensions==4.14.0 +tzdata @ file:///rapids/tzdata-2025.2-py2.py3-none-any.whl#sha256=1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 +ucx-py @ file:///rapids/ucx_py-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=4a452fd8d8c983283ada3d49086e3657dc2ee94fbad08efdd708d15ab6e0b73a +ucxx @ file:///rapids/ucxx-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=f1b4e55cb45c66fe88a2d8f9c2b87751165f3cfec21fbdc9c47c4ddff75e8a5f +-e git+https://github.com/Dots-Infra/UltraEP.git@94cab099b44fffa99a82fea99e7c12d89cf65e4f#egg=ultra_ep +uri-template==1.3.0 +urllib3==2.0.7 +wasabi==1.1.3 +wcwidth==0.2.13 +weasel==0.4.1 +webcolors==24.11.1 +webencodings==0.5.1 +websocket-client==1.8.0 +Werkzeug==3.1.3 +wheel==0.45.1 +wrapt==1.17.2 +xdoctest==1.0.2 +xgboost @ file:///rapids/xgboost-2.1.4-py3-none-manylinux_2_39_x86_64.whl#sha256=25eeca27f4048fc491940b3a0b4b5782ee48c33cb3aec059cef71f8da542930b +xxhash==4.0.1 +yarl @ file:///rapids/yarl-1.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=bf099e2432131093cc611623e0b0bcc399b8cddd9a91eded8bfb50402ec35018 +zarr @ file:///rapids/zarr-2.18.7-py3-none-any.whl#sha256=ac3dc4033e9ae4e9d7b5e27c97ea3eaf1003cc0a07f010bd83d5134bf8c4b223 +zict @ file:///rapids/zict-3.0.0-py2.py3-none-any.whl#sha256=5796e36bd0e0cc8cf0fbc1ace6a68912611c1dbd74750a3f3026b9b9d6a327ae +zipp==3.22.0 diff --git a/moonep/tierbmem/provenance.txt b/moonep/tierbmem/provenance.txt new file mode 100644 index 0000000000000000000000000000000000000000..2066c9496ae2e8b5602219f870a057bffcfa7c20 --- /dev/null +++ b/moonep/tierbmem/provenance.txt @@ -0,0 +1,90 @@ +============================================================== +PROVENANCE: moonep__tierbmem__gbs8 +============================================================== +captured_at : 2026-08-26T19:29:36+00:00 +hostname : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 +kernel : 6.8.0-1043-nvidia +cwd : /workspace/blackwell_run_scripts +command : run_ladder.sh --phase tierbmem --gbs 8 --out /workspace/tierb_study_20260826_144644/moonep/tierbmem --nsys + +---- GPU / driver -------------------------------------------- + 0, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 1, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 2, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 3, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 4, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 5, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 6, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 7, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + persistence/clocks: + 0, 120 MHz, 1965 MHz, 1000.00 W, 29 + 1, 165 MHz, 1965 MHz, 1000.00 W, 34 + 2, 157 MHz, 1965 MHz, 1000.00 W, 33 + 3, 210 MHz, 1965 MHz, 1000.00 W, 30 + 4, 330 MHz, 1965 MHz, 1000.00 W, 30 + 5, 195 MHz, 1965 MHz, 1000.00 W, 34 + 6, 210 MHz, 1965 MHz, 1000.00 W, 33 + 7, 120 MHz, 1965 MHz, 1000.00 W, 29 + + NVLink topology: + GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 CPU Affinity NUMA Affinity GPU NUMA ID + GPU0 X NV18 NV18 NV18 NV18 NV18 NV18 NV18 PIX NODE SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU1 NV18 X NV18 NV18 NV18 NV18 NV18 NV18 NODE PIX SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU2 NV18 NV18 X NV18 NV18 NV18 NV18 NV18 SYS SYS PIX NODE SYS SYS SYS SYS SYS 28-55 1 N/A + GPU3 NV18 NV18 NV18 X NV18 NV18 NV18 NV18 SYS SYS NODE PIX SYS SYS SYS SYS SYS 28-55 1 N/A + GPU4 NV18 NV18 NV18 NV18 X NV18 NV18 NV18 SYS SYS SYS SYS PIX NODE SYS SYS SYS 56-83 2 N/A + GPU5 NV18 NV18 NV18 NV18 NV18 X NV18 NV18 SYS SYS SYS SYS NODE PIX SYS SYS SYS 56-83 2 N/A + GPU6 NV18 NV18 NV18 NV18 NV18 NV18 X NV18 SYS SYS SYS SYS SYS SYS PIX NODE SYS 84-111 3 N/A + GPU7 NV18 NV18 NV18 NV18 NV18 NV18 NV18 X SYS SYS SYS SYS SYS SYS NODE PIX SYS 84-111 3 N/A + NIC0 PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS SYS NODE + NIC1 NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS SYS NODE + NIC2 SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS + NIC3 SYS SYS NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS + NIC4 SYS SYS SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS + unavailable + +---- toolchain ----------------------------------------------- + nvcc : release 12.9, V12.9.86 + gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 + python : 3.12.3 + torch : 2.8.0a0+5228986c39.nv25.06 cuda=12.9 nccl=2.27.3 + torch arch : ['sm_75', 'sm_80', 'sm_86', 'sm_90', 'sm_100', 'sm_120', 'compute_120'] + transformer_engine: 2.10.0.dev0+10f6128d [/usr/local/lib/python3.12/dist-packages/transformer_engine/__init__.py] + deep_ep : 1.2.1+34152ae [/workspace/DeepEP/deep_ep/__init__.py] + ultra_ep : 1.0.0 [/workspace/UltraEP/ultra_ep/__init__.py] + moonep : 0.0.1 [/workspace/MoonEP/moonep/__init__.py] + megatron : 0.14.0rc5 [None] + flash_attn : 2.7.4.post1 [/usr/local/lib/python3.12/dist-packages/flash_attn/__init__.py] + numpy : 1.26.4 [/usr/local/lib/python3.12/dist-packages/numpy/__init__.py] + huggingface_hub: 1.28.0 [/usr/local/lib/python3.12/dist-packages/huggingface_hub/__init__.py] + +---- repositories -------------------------------------------- + Megatron-MoE-Echo moonep_backend_better_memory_dump_support 453b84c7376f + Megatron-UltraEP ultraep-better-memory-dump-support c3a8a8a7b69d + UltraEP main 94cab099b44f + DeepEP HEAD 34152ae28f80 (+3 untracked: build output) + MoonEP megatron-integration c27d35da15f2 + blackwell_run_scripts granularity-moonep ef3305ebfabd + +---- environment --------------------------------------------- + CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0 + CUDA_DRIVER_VERSION=575.57.08 + CUDA_HOME=/usr/local/cuda + CUDA_MODULE_LOADING=LAZY + CUDA_VERSION=12.9.1.010 + HF_REMOTE_ROOT=tierb_study_20260826_144644 + NCCL_VERSION=2.27.3 + NVSHMEM_VERSION=3.2.5 + PYTORCH_BUILD_NUMBER=0 + PYTORCH_BUILD_VERSION=2.8.0a0+5228986 + PYTORCH_HOME=/opt/pytorch/pytorch + PYTORCH_VERSION=2.8.0a0+5228986 + TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1 + TORCH_CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0+PTX + TORCH_NCCL_USE_COMM_NONBLOCKING=0 + +---- shared library resolution ------------------------------- + libnvshmem_host.so: not in ldconfig + libnccl.so: /lib/x86_64-linux-gnu/libnccl.so.2 + lrwxrwxrwx 1 root root 20 Aug 26 14:37 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so -> libnvshmem_host.so.3 + -rw-r--r-- 1 root root 179045744 Aug 26 14:09 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so.3 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/mem_samples.csv b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..db4add2345816fce233bd1037e093689aa30f994 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/mem_samples.csv @@ -0,0 +1,208 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 819 +6, 821 +7, 819 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 2471 +1, 2471 +2, 2471 +3, 2471 +4, 2471 +5, 2471 +6, 2471 +7, 2471 +0, 5455 +1, 5455 +2, 5455 +3, 5455 +4, 5455 +5, 5455 +6, 5455 +7, 5455 +0, 35381 +1, 35505 +2, 35375 +3, 35435 +4, 35387 +5, 35435 +6, 35423 +7, 35505 +0, 41335 +1, 41457 +2, 41331 +3, 41389 +4, 41341 +5, 41387 +6, 41377 +7, 41457 +0, 46291 +1, 46411 +2, 46285 +3, 46343 +4, 46295 +5, 46341 +6, 46333 +7, 46411 +0, 46291 +1, 46411 +2, 46285 +3, 46343 +4, 46295 +5, 46341 +6, 46333 +7, 46411 +0, 46333 +1, 46453 +2, 46319 +3, 46387 +4, 46339 +5, 46385 +6, 46375 +7, 46455 +0, 138765 +1, 138757 +2, 138759 +3, 138755 +4, 138771 +5, 138751 +6, 139529 +7, 138757 +0, 139659 +1, 139629 +2, 139627 +3, 139635 +4, 139661 +5, 139647 +6, 140423 +7, 139689 +0, 139801 +1, 139793 +2, 139795 +3, 139789 +4, 139805 +5, 139787 +6, 140565 +7, 139793 +0, 142499 +1, 142491 +2, 142493 +3, 142487 +4, 142503 +5, 142485 +6, 143263 +7, 142491 +0, 143063 +1, 143057 +2, 143089 +3, 143101 +4, 143089 +5, 143091 +6, 143829 +7, 143035 +0, 90449 +1, 90435 +2, 90447 +3, 90461 +4, 90453 +5, 90497 +6, 90429 +7, 90457 +0, 165291 +1, 165281 +2, 165285 +3, 165279 +4, 165295 +5, 165275 +6, 165269 +7, 165281 +0, 165291 +1, 165281 +2, 165285 +3, 165279 +4, 165295 +5, 165275 +6, 165269 +7, 165281 +0, 11296 +1, 165077 +2, 165081 +3, 165075 +4, 165091 +5, 165071 +6, 165065 +7, 165077 +0, 9373 +1, 9695 +2, 165081 +3, 10976 +4, 9376 +5, 9376 +6, 9376 +7, 9376 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T193044093906Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T193044093906Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aecccf6d4e3bb2ea0ae8d7b95a37509f0274b0b6 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T193044093906Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69357150208, + "active_bytes_current_delta": 45559357440, + "active_bytes_peak": 131018615296, + "active_bytes_peak_delta": 107208239616, + "after": { + "active_bytes_current": 69357150208, + "active_bytes_peak": 131018615296, + "allocated_bytes_current": 69357150208, + "allocated_bytes_peak": 131018615296, + "device": 0, + "inactive_split_bytes_current": 8940019712, + "inactive_split_bytes_peak": 22467192832, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 133695537152, + "reserved_bytes_peak": 133695537152, + "segment_count_current": 1121, + "segment_count_peak": 1121, + "timestamp_utc": "2026-08-26T19:30:44.093427Z", + "world_size": 8 + }, + "allocated_bytes_current": 69357150208, + "allocated_bytes_current_delta": 45559357440, + "allocated_bytes_peak": 131018615296, + "allocated_bytes_peak_delta": 107208239616, + "before": { + "active_bytes_current": 23797792768, + "active_bytes_peak": 23810375680, + "allocated_bytes_current": 23797792768, + "allocated_bytes_peak": 23810375680, + "device": 0, + "inactive_split_bytes_current": 768245760, + "inactive_split_bytes_peak": 1283983872, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 24582815744, + "reserved_bytes_peak": 24582815744, + "segment_count_current": 470, + "segment_count_peak": 470, + "timestamp_utc": "2026-08-26T19:30:21.878879Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8940019712, + "inactive_split_bytes_current_delta": 8171773952, + "inactive_split_bytes_peak": 22467192832, + "inactive_split_bytes_peak_delta": 21183208960, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 133695537152, + "reserved_bytes_current_delta": 109112721408, + "reserved_bytes_peak": 133695537152, + "reserved_bytes_peak_delta": 109112721408, + "segment_count_current": 1121, + "segment_count_current_delta": 651, + "segment_count_peak": 1121, + "segment_count_peak_delta": 651, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T193044093906Z.pickle", + "timestamp_utc": "2026-08-26T19:30:44.200427Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T193044093906Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T193044093906Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aa3d7f68f6bd38132568c1ec5e6fe83284c74112 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T193044093906Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4b81dc09917e790182d37f49a0415fc222d79905df3aefb1237a3f8c73556f9 +size 4773180 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T193044942835Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T193044942835Z.json new file mode 100644 index 0000000000000000000000000000000000000000..abf5b20ff4e647465d2cdd3176f24d58a1969d6c --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T193044942835Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69357150208, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145246385152, + "active_bytes_peak_delta": 14227769856, + "after": { + "active_bytes_current": 69357150208, + "active_bytes_peak": 145246385152, + "allocated_bytes_current": 69357150208, + "allocated_bytes_peak": 145246385152, + "device": 0, + "inactive_split_bytes_current": 8940019712, + "inactive_split_bytes_peak": 22467192832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 148241383424, + "reserved_bytes_peak": 148241383424, + "segment_count_current": 1134, + "segment_count_peak": 1134, + "timestamp_utc": "2026-08-26T19:30:44.942419Z", + "world_size": 8 + }, + "allocated_bytes_current": 69357150208, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145246385152, + "allocated_bytes_peak_delta": 14227769856, + "before": { + "active_bytes_current": 69357150208, + "active_bytes_peak": 131018615296, + "allocated_bytes_current": 69357150208, + "allocated_bytes_peak": 131018615296, + "device": 0, + "inactive_split_bytes_current": 8940019712, + "inactive_split_bytes_peak": 22467192832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 133695537152, + "reserved_bytes_peak": 133695537152, + "segment_count_current": 1121, + "segment_count_peak": 1121, + "timestamp_utc": "2026-08-26T19:30:44.208120Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8940019712, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22467192832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 148241383424, + "reserved_bytes_current_delta": 14545846272, + "reserved_bytes_peak": 148241383424, + "reserved_bytes_peak_delta": 14545846272, + "segment_count_current": 1134, + "segment_count_current_delta": 13, + "segment_count_peak": 1134, + "segment_count_peak_delta": 13, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T193044942835Z.pickle", + "timestamp_utc": "2026-08-26T19:30:45.022806Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T193044942835Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T193044942835Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..db00612128c375dff0b00914c232850589726759 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T193044942835Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1444c25828f4fb76a1709547998722e4c0e9a71487e44211d6df2bdba183bab7 +size 3999220 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T193045765092Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T193045765092Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f14c58774f34d41de74dd983b16f271c83dffa79 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T193045765092Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69357150208, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145246385152, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69357150208, + "active_bytes_peak": 145246385152, + "allocated_bytes_current": 69357150208, + "allocated_bytes_peak": 145246385152, + "device": 0, + "inactive_split_bytes_current": 8940019712, + "inactive_split_bytes_peak": 22467192832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 148241383424, + "reserved_bytes_peak": 148241383424, + "segment_count_current": 1134, + "segment_count_peak": 1134, + "timestamp_utc": "2026-08-26T19:30:45.764700Z", + "world_size": 8 + }, + "allocated_bytes_current": 69357150208, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145246385152, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69357150208, + "active_bytes_peak": 145246385152, + "allocated_bytes_current": 69357150208, + "allocated_bytes_peak": 145246385152, + "device": 0, + "inactive_split_bytes_current": 8940019712, + "inactive_split_bytes_peak": 22467192832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 148241383424, + "reserved_bytes_peak": 148241383424, + "segment_count_current": 1134, + "segment_count_peak": 1134, + "timestamp_utc": "2026-08-26T19:30:45.028867Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8940019712, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22467192832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 148241383424, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148241383424, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1134, + "segment_count_current_delta": 0, + "segment_count_peak": 1134, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T193045765092Z.pickle", + "timestamp_utc": "2026-08-26T19:30:45.842119Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T193045765092Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T193045765092Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ae4723613473c5e500bf2a91642c4fb6f37923e2 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T193045765092Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:434bf7b84fee436346c7573553907450c4aa942a8449acee2b589c10cdf9960a +size 3999201 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T193047892493Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T193047892493Z.json new file mode 100644 index 0000000000000000000000000000000000000000..76c0b57f568b36894e8d3381a744e8ed7330450a --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T193047892493Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69375111168, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 145246385152, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69375111168, + "active_bytes_peak": 145246385152, + "allocated_bytes_current": 69375111168, + "allocated_bytes_peak": 145246385152, + "device": 0, + "inactive_split_bytes_current": 11115679744, + "inactive_split_bytes_peak": 22467192832, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 156734849024, + "reserved_bytes_peak": 156734849024, + "segment_count_current": 1231, + "segment_count_peak": 1231, + "timestamp_utc": "2026-08-26T19:30:47.892052Z", + "world_size": 8 + }, + "allocated_bytes_current": 69375111168, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 145246385152, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69357150208, + "active_bytes_peak": 145246385152, + "allocated_bytes_current": 69357150208, + "allocated_bytes_peak": 145246385152, + "device": 0, + "inactive_split_bytes_current": 8940019712, + "inactive_split_bytes_peak": 22467192832, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 148241383424, + "reserved_bytes_peak": 148241383424, + "segment_count_current": 1134, + "segment_count_peak": 1134, + "timestamp_utc": "2026-08-26T19:30:45.848071Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 11115679744, + "inactive_split_bytes_current_delta": 2175660032, + "inactive_split_bytes_peak": 22467192832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 156734849024, + "reserved_bytes_current_delta": 8493465600, + "reserved_bytes_peak": 156734849024, + "reserved_bytes_peak_delta": 8493465600, + "segment_count_current": 1231, + "segment_count_current_delta": 97, + "segment_count_peak": 1231, + "segment_count_peak_delta": 97, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T193047892493Z.pickle", + "timestamp_utc": "2026-08-26T19:30:47.983246Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T193047892493Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T193047892493Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e8c964485817b88bdeb62ab3a8d517c82d490b8b --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T193047892493Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e7c44623d98e5396db1985e6b1c9c7771ab16af5e9796fc3d38d081365ffed0 +size 4107929 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T193048707005Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T193048707005Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6318381e43d03416ad6c903e3caf107a528e341f --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T193048707005Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69375111168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145246385152, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69375111168, + "active_bytes_peak": 145246385152, + "allocated_bytes_current": 69375111168, + "allocated_bytes_peak": 145246385152, + "device": 0, + "inactive_split_bytes_current": 11115679744, + "inactive_split_bytes_peak": 22467192832, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 156734849024, + "reserved_bytes_peak": 156734849024, + "segment_count_current": 1231, + "segment_count_peak": 1231, + "timestamp_utc": "2026-08-26T19:30:48.706609Z", + "world_size": 8 + }, + "allocated_bytes_current": 69375111168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145246385152, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69375111168, + "active_bytes_peak": 145246385152, + "allocated_bytes_current": 69375111168, + "allocated_bytes_peak": 145246385152, + "device": 0, + "inactive_split_bytes_current": 11115679744, + "inactive_split_bytes_peak": 22467192832, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 156734849024, + "reserved_bytes_peak": 156734849024, + "segment_count_current": 1231, + "segment_count_peak": 1231, + "timestamp_utc": "2026-08-26T19:30:47.990002Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 11115679744, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22467192832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 156734849024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 156734849024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1231, + "segment_count_current_delta": 0, + "segment_count_peak": 1231, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T193048707005Z.pickle", + "timestamp_utc": "2026-08-26T19:30:48.753410Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T193048707005Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T193048707005Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..43768c7d053d61bf7957c84cda1c9a914aeee6a7 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T193048707005Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f77ff29aee38821a1118bc9ea31a148eced492438dce2a1d9245fd94fb83a4b +size 2018041 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T193049474961Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T193049474961Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0389cb3c463f5895c489087abc54b60768abd408 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T193049474961Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69375111168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145246385152, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69375111168, + "active_bytes_peak": 145246385152, + "allocated_bytes_current": 69375111168, + "allocated_bytes_peak": 145246385152, + "device": 0, + "inactive_split_bytes_current": 11115679744, + "inactive_split_bytes_peak": 22467192832, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 156734849024, + "reserved_bytes_peak": 156734849024, + "segment_count_current": 1231, + "segment_count_peak": 1231, + "timestamp_utc": "2026-08-26T19:30:49.474569Z", + "world_size": 8 + }, + "allocated_bytes_current": 69375111168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145246385152, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69375111168, + "active_bytes_peak": 145246385152, + "allocated_bytes_current": 69375111168, + "allocated_bytes_peak": 145246385152, + "device": 0, + "inactive_split_bytes_current": 11115679744, + "inactive_split_bytes_peak": 22467192832, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 156734849024, + "reserved_bytes_peak": 156734849024, + "segment_count_current": 1231, + "segment_count_peak": 1231, + "timestamp_utc": "2026-08-26T19:30:48.755830Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 11115679744, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22467192832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 156734849024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 156734849024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1231, + "segment_count_current_delta": 0, + "segment_count_peak": 1231, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260826T193049474961Z.pickle", + "timestamp_utc": "2026-08-26T19:30:49.519940Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T193049474961Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T193049474961Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f49c1aa1cef238a72765d1a5baa9c6edd2d95858 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260826T193049474961Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02482015e0777c18f5be10f5d946f74001020f92cfbbca55b0fb1ab44f38fd3d +size 2018041 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T193044093907Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T193044093907Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8eb6b5656bba4e77a256e3bac363675904a81568 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T193044093907Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69366574080, + "active_bytes_current_delta": 45563923456, + "active_bytes_peak": 131036168704, + "active_bytes_peak_delta": 107220935168, + "after": { + "active_bytes_current": 69366574080, + "active_bytes_peak": 131036168704, + "allocated_bytes_current": 69366574080, + "allocated_bytes_peak": 131036168704, + "device": 1, + "inactive_split_bytes_current": 8920110080, + "inactive_split_bytes_peak": 22444142592, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 133687148544, + "reserved_bytes_peak": 133687148544, + "segment_count_current": 1127, + "segment_count_peak": 1127, + "timestamp_utc": "2026-08-26T19:30:44.093394Z", + "world_size": 8 + }, + "allocated_bytes_current": 69366574080, + "allocated_bytes_current_delta": 45563923456, + "allocated_bytes_peak": 131036168704, + "allocated_bytes_peak_delta": 107220935168, + "before": { + "active_bytes_current": 23802650624, + "active_bytes_peak": 23815233536, + "allocated_bytes_current": 23802650624, + "allocated_bytes_peak": 23815233536, + "device": 1, + "inactive_split_bytes_current": 891314176, + "inactive_split_bytes_peak": 1283983872, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 24712839168, + "reserved_bytes_peak": 24712839168, + "segment_count_current": 477, + "segment_count_peak": 477, + "timestamp_utc": "2026-08-26T19:30:21.878815Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8920110080, + "inactive_split_bytes_current_delta": 8028795904, + "inactive_split_bytes_peak": 22444142592, + "inactive_split_bytes_peak_delta": 21160158720, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 133687148544, + "reserved_bytes_current_delta": 108974309376, + "reserved_bytes_peak": 133687148544, + "reserved_bytes_peak_delta": 108974309376, + "segment_count_current": 1127, + "segment_count_current_delta": 650, + "segment_count_peak": 1127, + "segment_count_peak_delta": 650, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T193044093907Z.pickle", + "timestamp_utc": "2026-08-26T19:30:44.201583Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T193044093907Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T193044093907Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6c75b4bb0d3ce099c4b1e7772fb55a255520b577 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T193044093907Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7eae6f23347b618da0a39e463bdf7865536e9b8ae1151c0516f7876f75b83a91 +size 4745059 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T193044942950Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T193044942950Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e59c71fbe31c1e5d69fa677d4070eee0f2d7ccf7 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T193044942950Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69366574080, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145263830528, + "active_bytes_peak_delta": 14227661824, + "after": { + "active_bytes_current": 69366574080, + "active_bytes_peak": 145263830528, + "allocated_bytes_current": 69366574080, + "allocated_bytes_peak": 145263830528, + "device": 1, + "inactive_split_bytes_current": 8920110080, + "inactive_split_bytes_peak": 22444142592, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:30:44.942514Z", + "world_size": 8 + }, + "allocated_bytes_current": 69366574080, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145263830528, + "allocated_bytes_peak_delta": 14227661824, + "before": { + "active_bytes_current": 69366574080, + "active_bytes_peak": 131036168704, + "allocated_bytes_current": 69366574080, + "allocated_bytes_peak": 131036168704, + "device": 1, + "inactive_split_bytes_current": 8920110080, + "inactive_split_bytes_peak": 22444142592, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 133687148544, + "reserved_bytes_peak": 133687148544, + "segment_count_current": 1127, + "segment_count_peak": 1127, + "timestamp_utc": "2026-08-26T19:30:44.209908Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8920110080, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22444142592, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 148226703360, + "reserved_bytes_current_delta": 14539554816, + "reserved_bytes_peak": 148226703360, + "reserved_bytes_peak_delta": 14539554816, + "segment_count_current": 1137, + "segment_count_current_delta": 10, + "segment_count_peak": 1137, + "segment_count_peak_delta": 10, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T193044942950Z.pickle", + "timestamp_utc": "2026-08-26T19:30:45.030697Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T193044942950Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T193044942950Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f666a4340debcc1266ebe315ed20eb039e01c89b --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T193044942950Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92a6d47a38381f250b6ce85d894826a709b00a1255d7b0d4098bb33ba5a4ae1c +size 3972684 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T193045765223Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T193045765223Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e0d937864f63f9bde88f2695f43066bf2b8fe1f9 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T193045765223Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69366574080, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145263830528, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69366574080, + "active_bytes_peak": 145263830528, + "allocated_bytes_current": 69366574080, + "allocated_bytes_peak": 145263830528, + "device": 1, + "inactive_split_bytes_current": 8920110080, + "inactive_split_bytes_peak": 22444142592, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:30:45.764790Z", + "world_size": 8 + }, + "allocated_bytes_current": 69366574080, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145263830528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69366574080, + "active_bytes_peak": 145263830528, + "allocated_bytes_current": 69366574080, + "allocated_bytes_peak": 145263830528, + "device": 1, + "inactive_split_bytes_current": 8920110080, + "inactive_split_bytes_peak": 22444142592, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:30:45.037782Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8920110080, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22444142592, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 148226703360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148226703360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1137, + "segment_count_current_delta": 0, + "segment_count_peak": 1137, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T193045765223Z.pickle", + "timestamp_utc": "2026-08-26T19:30:45.848367Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T193045765223Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T193045765223Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e7ae325b59058b32cc75003da63a21fd57772d25 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T193045765223Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5305aae8b05cac6e4b127a987d91bbc32bbd854029283f2bb3ebbbb7fb9f2a4d +size 3972950 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T193047892626Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T193047892626Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cc19af1a1d5dd717fad0f653a19c3d3f95927853 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T193047892626Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69384535040, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 145263830528, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69384535040, + "active_bytes_peak": 145263830528, + "allocated_bytes_current": 69384535040, + "allocated_bytes_peak": 145263830528, + "device": 1, + "inactive_split_bytes_current": 11095770112, + "inactive_split_bytes_peak": 22444142592, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 156724363264, + "reserved_bytes_peak": 156724363264, + "segment_count_current": 1236, + "segment_count_peak": 1236, + "timestamp_utc": "2026-08-26T19:30:47.892135Z", + "world_size": 8 + }, + "allocated_bytes_current": 69384535040, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 145263830528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69366574080, + "active_bytes_peak": 145263830528, + "allocated_bytes_current": 69366574080, + "allocated_bytes_peak": 145263830528, + "device": 1, + "inactive_split_bytes_current": 8920110080, + "inactive_split_bytes_peak": 22444142592, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:30:45.854776Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 11095770112, + "inactive_split_bytes_current_delta": 2175660032, + "inactive_split_bytes_peak": 22444142592, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 156724363264, + "reserved_bytes_current_delta": 8497659904, + "reserved_bytes_peak": 156724363264, + "reserved_bytes_peak_delta": 8497659904, + "segment_count_current": 1236, + "segment_count_current_delta": 99, + "segment_count_peak": 1236, + "segment_count_peak_delta": 99, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T193047892626Z.pickle", + "timestamp_utc": "2026-08-26T19:30:47.990951Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T193047892626Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T193047892626Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5d26b12e21f1ef35a84dfd3467b1f292eea8f874 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T193047892626Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f827f8cacdfdb225ce34655cf8d667a85abc6bc0e9941e74e17de99d550a48c +size 4081535 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T193048707113Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T193048707113Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3fd85130ae937c95b5f7ca58da7805977c50a345 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T193048707113Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69384535040, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145263830528, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69384535040, + "active_bytes_peak": 145263830528, + "allocated_bytes_current": 69384535040, + "allocated_bytes_peak": 145263830528, + "device": 1, + "inactive_split_bytes_current": 11095770112, + "inactive_split_bytes_peak": 22444142592, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 156724363264, + "reserved_bytes_peak": 156724363264, + "segment_count_current": 1236, + "segment_count_peak": 1236, + "timestamp_utc": "2026-08-26T19:30:48.706700Z", + "world_size": 8 + }, + "allocated_bytes_current": 69384535040, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145263830528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69384535040, + "active_bytes_peak": 145263830528, + "allocated_bytes_current": 69384535040, + "allocated_bytes_peak": 145263830528, + "device": 1, + "inactive_split_bytes_current": 11095770112, + "inactive_split_bytes_peak": 22444142592, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 156724363264, + "reserved_bytes_peak": 156724363264, + "segment_count_current": 1236, + "segment_count_peak": 1236, + "timestamp_utc": "2026-08-26T19:30:47.998302Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 11095770112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22444142592, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 156724363264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 156724363264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1236, + "segment_count_current_delta": 0, + "segment_count_peak": 1236, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T193048707113Z.pickle", + "timestamp_utc": "2026-08-26T19:30:48.757654Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T193048707113Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T193048707113Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..49544ae572854f03b6af13bd898c2a5710ec6135 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T193048707113Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:547f67b2377178a53038b5e942ffc25cf41edebb25a629c372089a3bf4bf2958 +size 2005705 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T193049475067Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T193049475067Z.json new file mode 100644 index 0000000000000000000000000000000000000000..52b41d7b1bca6d8a5c47f3768ebf0d057468747c --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T193049475067Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69384535040, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145263830528, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69384535040, + "active_bytes_peak": 145263830528, + "allocated_bytes_current": 69384535040, + "allocated_bytes_peak": 145263830528, + "device": 1, + "inactive_split_bytes_current": 11095770112, + "inactive_split_bytes_peak": 22444142592, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 156724363264, + "reserved_bytes_peak": 156724363264, + "segment_count_current": 1236, + "segment_count_peak": 1236, + "timestamp_utc": "2026-08-26T19:30:49.474675Z", + "world_size": 8 + }, + "allocated_bytes_current": 69384535040, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145263830528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69384535040, + "active_bytes_peak": 145263830528, + "allocated_bytes_current": 69384535040, + "allocated_bytes_peak": 145263830528, + "device": 1, + "inactive_split_bytes_current": 11095770112, + "inactive_split_bytes_peak": 22444142592, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 156724363264, + "reserved_bytes_peak": 156724363264, + "segment_count_current": 1236, + "segment_count_peak": 1236, + "timestamp_utc": "2026-08-26T19:30:48.760119Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 11095770112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22444142592, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 156724363264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 156724363264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1236, + "segment_count_current_delta": 0, + "segment_count_peak": 1236, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T193049475067Z.pickle", + "timestamp_utc": "2026-08-26T19:30:49.521811Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T193049475067Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T193049475067Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9223a805e859d4bc52001e2be248b1c23e67d1f9 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T193049475067Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9d2f23a8c73dc2859df52009b01ba2feb740539681052b05799c272633c2fa7 +size 2005705 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T193044093907Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T193044093907Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c213a39473ed818c4ba3515ffc73407e3babcf2a --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T193044093907Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69357210624, + "active_bytes_current_delta": 45549528064, + "active_bytes_peak": 131032331776, + "active_bytes_peak_delta": 107212066304, + "after": { + "active_bytes_current": 69357210624, + "active_bytes_peak": 131032331776, + "allocated_bytes_current": 69357210624, + "allocated_bytes_peak": 131032331776, + "device": 2, + "inactive_split_bytes_current": 8933667840, + "inactive_split_bytes_peak": 22455555072, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 133689245696, + "reserved_bytes_peak": 133689245696, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:30:44.093409Z", + "world_size": 8 + }, + "allocated_bytes_current": 69357210624, + "allocated_bytes_current_delta": 45549528064, + "allocated_bytes_peak": 131032331776, + "allocated_bytes_peak_delta": 107212066304, + "before": { + "active_bytes_current": 23807682560, + "active_bytes_peak": 23820265472, + "allocated_bytes_current": 23807682560, + "allocated_bytes_peak": 23820265472, + "device": 2, + "inactive_split_bytes_current": 754161664, + "inactive_split_bytes_peak": 1283983872, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 24576524288, + "reserved_bytes_peak": 24576524288, + "segment_count_current": 472, + "segment_count_peak": 472, + "timestamp_utc": "2026-08-26T19:30:21.879008Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8933667840, + "inactive_split_bytes_current_delta": 8179506176, + "inactive_split_bytes_peak": 22455555072, + "inactive_split_bytes_peak_delta": 21171571200, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 133689245696, + "reserved_bytes_current_delta": 109112721408, + "reserved_bytes_peak": 133689245696, + "reserved_bytes_peak_delta": 109112721408, + "segment_count_current": 1123, + "segment_count_current_delta": 651, + "segment_count_peak": 1123, + "segment_count_peak_delta": 651, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T193044093907Z.pickle", + "timestamp_utc": "2026-08-26T19:30:44.204306Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T193044093907Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T193044093907Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4eaf364e122fc5921c0a65ebc0ff3203230c6973 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T193044093907Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a9d0a7a2ee124d1bbb6d2d8d986f77cc549b7eb41348871724b25f061b817bc +size 4756981 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T193044942866Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T193044942866Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0dd5016a368b0c46dcef987f018d394be7b9613d --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T193044942866Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69357210624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145251977216, + "active_bytes_peak_delta": 14219645440, + "after": { + "active_bytes_current": 69357210624, + "active_bytes_peak": 145251977216, + "allocated_bytes_current": 69357210624, + "allocated_bytes_peak": 145251977216, + "device": 2, + "inactive_split_bytes_current": 8933667840, + "inactive_split_bytes_peak": 22455555072, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 148235091968, + "reserved_bytes_peak": 148235091968, + "segment_count_current": 1136, + "segment_count_peak": 1136, + "timestamp_utc": "2026-08-26T19:30:44.942458Z", + "world_size": 8 + }, + "allocated_bytes_current": 69357210624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145251977216, + "allocated_bytes_peak_delta": 14219645440, + "before": { + "active_bytes_current": 69357210624, + "active_bytes_peak": 131032331776, + "allocated_bytes_current": 69357210624, + "allocated_bytes_peak": 131032331776, + "device": 2, + "inactive_split_bytes_current": 8933667840, + "inactive_split_bytes_peak": 22455555072, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 133689245696, + "reserved_bytes_peak": 133689245696, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:30:44.213564Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8933667840, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22455555072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 148235091968, + "reserved_bytes_current_delta": 14545846272, + "reserved_bytes_peak": 148235091968, + "reserved_bytes_peak_delta": 14545846272, + "segment_count_current": 1136, + "segment_count_current_delta": 13, + "segment_count_peak": 1136, + "segment_count_peak_delta": 13, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T193044942866Z.pickle", + "timestamp_utc": "2026-08-26T19:30:45.033959Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T193044942866Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T193044942866Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6ae90118a529d4acf5e15930ab0d042ddd630323 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T193044942866Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95260a4bcfc840ee6efea9c3e2f6cfabf185a19c79817fdc29247ec6613a1d5c +size 3980988 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T193045765135Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T193045765135Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6c70c32bb970cc55ce12b6966428c5fab0c17e1c --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T193045765135Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69357210624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145252370432, + "active_bytes_peak_delta": 393216, + "after": { + "active_bytes_current": 69357210624, + "active_bytes_peak": 145252370432, + "allocated_bytes_current": 69357210624, + "allocated_bytes_peak": 145252370432, + "device": 2, + "inactive_split_bytes_current": 8933667840, + "inactive_split_bytes_peak": 22455555072, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 148235091968, + "reserved_bytes_peak": 148235091968, + "segment_count_current": 1136, + "segment_count_peak": 1136, + "timestamp_utc": "2026-08-26T19:30:45.764734Z", + "world_size": 8 + }, + "allocated_bytes_current": 69357210624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145252370432, + "allocated_bytes_peak_delta": 393216, + "before": { + "active_bytes_current": 69357210624, + "active_bytes_peak": 145251977216, + "allocated_bytes_current": 69357210624, + "allocated_bytes_peak": 145251977216, + "device": 2, + "inactive_split_bytes_current": 8933667840, + "inactive_split_bytes_peak": 22455555072, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 148235091968, + "reserved_bytes_peak": 148235091968, + "segment_count_current": 1136, + "segment_count_peak": 1136, + "timestamp_utc": "2026-08-26T19:30:45.041343Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8933667840, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22455555072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 148235091968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148235091968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1136, + "segment_count_current_delta": 0, + "segment_count_peak": 1136, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T193045765135Z.pickle", + "timestamp_utc": "2026-08-26T19:30:45.851783Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T193045765135Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T193045765135Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fdd10486d95335611e843f0bf55418eed8c3fdda --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T193045765135Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3a2b556a09f84162c93908969d4bd240f64d69452c39b930bc87f4e4d9449ea +size 3981143 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T193047892582Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T193047892582Z.json new file mode 100644 index 0000000000000000000000000000000000000000..119e4498ae142062b501f1158d0f45bd4b8b4f92 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T193047892582Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69375171584, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 145252370432, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69375171584, + "active_bytes_peak": 145252370432, + "allocated_bytes_current": 69375171584, + "allocated_bytes_peak": 145252370432, + "device": 2, + "inactive_split_bytes_current": 11109327872, + "inactive_split_bytes_peak": 22455555072, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 156728557568, + "reserved_bytes_peak": 156728557568, + "segment_count_current": 1233, + "segment_count_peak": 1233, + "timestamp_utc": "2026-08-26T19:30:47.892106Z", + "world_size": 8 + }, + "allocated_bytes_current": 69375171584, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 145252370432, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69357210624, + "active_bytes_peak": 145252370432, + "allocated_bytes_current": 69357210624, + "allocated_bytes_peak": 145252370432, + "device": 2, + "inactive_split_bytes_current": 8933667840, + "inactive_split_bytes_peak": 22455555072, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 148235091968, + "reserved_bytes_peak": 148235091968, + "segment_count_current": 1136, + "segment_count_peak": 1136, + "timestamp_utc": "2026-08-26T19:30:45.859207Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 11109327872, + "inactive_split_bytes_current_delta": 2175660032, + "inactive_split_bytes_peak": 22455555072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 156728557568, + "reserved_bytes_current_delta": 8493465600, + "reserved_bytes_peak": 156728557568, + "reserved_bytes_peak_delta": 8493465600, + "segment_count_current": 1233, + "segment_count_current_delta": 97, + "segment_count_peak": 1233, + "segment_count_peak_delta": 97, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T193047892582Z.pickle", + "timestamp_utc": "2026-08-26T19:30:47.990033Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T193047892582Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T193047892582Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b09f5ee741073895c153ddb7848753bc9a7dc451 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T193047892582Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f823b2f3986f366e1a311d055cf60a9a76085d3d011bfdb5af2d4a83804b1223 +size 4091410 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T193048707085Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T193048707085Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4061fa98f006f001e858ec8c10410268d3f28f9a --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T193048707085Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69375171584, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145252370432, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69375171584, + "active_bytes_peak": 145252370432, + "allocated_bytes_current": 69375171584, + "allocated_bytes_peak": 145252370432, + "device": 2, + "inactive_split_bytes_current": 11109327872, + "inactive_split_bytes_peak": 22455555072, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 156728557568, + "reserved_bytes_peak": 156728557568, + "segment_count_current": 1233, + "segment_count_peak": 1233, + "timestamp_utc": "2026-08-26T19:30:48.706676Z", + "world_size": 8 + }, + "allocated_bytes_current": 69375171584, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145252370432, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69375171584, + "active_bytes_peak": 145252370432, + "allocated_bytes_current": 69375171584, + "allocated_bytes_peak": 145252370432, + "device": 2, + "inactive_split_bytes_current": 11109327872, + "inactive_split_bytes_peak": 22455555072, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 156728557568, + "reserved_bytes_peak": 156728557568, + "segment_count_current": 1233, + "segment_count_peak": 1233, + "timestamp_utc": "2026-08-26T19:30:47.997793Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 11109327872, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22455555072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 156728557568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 156728557568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1233, + "segment_count_current_delta": 0, + "segment_count_peak": 1233, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T193048707085Z.pickle", + "timestamp_utc": "2026-08-26T19:30:48.760789Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T193048707085Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T193048707085Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0fbddc8c6637f0e629e8d48f844273fba5e04168 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T193048707085Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90b4ffb20c1d80749c6a3a62ef1bd2947a8fa0c64e5e080c91b43eb8b9f107e8 +size 2012302 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T193049475039Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T193049475039Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cd07e4a8f23fa87bb25b54b8de4501f19c7014cc --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T193049475039Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69375171584, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145252370432, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69375171584, + "active_bytes_peak": 145252370432, + "allocated_bytes_current": 69375171584, + "allocated_bytes_peak": 145252370432, + "device": 2, + "inactive_split_bytes_current": 11109327872, + "inactive_split_bytes_peak": 22455555072, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 156728557568, + "reserved_bytes_peak": 156728557568, + "segment_count_current": 1233, + "segment_count_peak": 1233, + "timestamp_utc": "2026-08-26T19:30:49.474653Z", + "world_size": 8 + }, + "allocated_bytes_current": 69375171584, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145252370432, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69375171584, + "active_bytes_peak": 145252370432, + "allocated_bytes_current": 69375171584, + "allocated_bytes_peak": 145252370432, + "device": 2, + "inactive_split_bytes_current": 11109327872, + "inactive_split_bytes_peak": 22455555072, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 156728557568, + "reserved_bytes_peak": 156728557568, + "segment_count_current": 1233, + "segment_count_peak": 1233, + "timestamp_utc": "2026-08-26T19:30:48.763239Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 11109327872, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22455555072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 156728557568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 156728557568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1233, + "segment_count_current_delta": 0, + "segment_count_peak": 1233, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T193049475039Z.pickle", + "timestamp_utc": "2026-08-26T19:30:49.526407Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T193049475039Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T193049475039Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..81a8f1fe05d0dc94e188df3a6d19e7dd4d3cc5be --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T193049475039Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13d7772304480f6c5846894d3df4ad96414fc44f9d8a2f62e71241c50fdae015 +size 2012302 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T193044093901Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T193044093901Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8c6f4b1598efc8aa7118701fd29827c53138977d --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T193044093901Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69357679104, + "active_bytes_current_delta": 45546421248, + "active_bytes_peak": 131031941632, + "active_bytes_peak_delta": 107208100864, + "after": { + "active_bytes_current": 69357679104, + "active_bytes_peak": 131031941632, + "allocated_bytes_current": 69357679104, + "allocated_bytes_peak": 131031941632, + "device": 3, + "inactive_split_bytes_current": 8926907904, + "inactive_split_bytes_peak": 22443462656, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 133682954240, + "reserved_bytes_peak": 133682954240, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:30:44.093371Z", + "world_size": 8 + }, + "allocated_bytes_current": 69357679104, + "allocated_bytes_current_delta": 45546421248, + "allocated_bytes_peak": 131031941632, + "allocated_bytes_peak_delta": 107208100864, + "before": { + "active_bytes_current": 23811257856, + "active_bytes_peak": 23823840768, + "allocated_bytes_current": 23811257856, + "allocated_bytes_peak": 23823840768, + "device": 3, + "inactive_split_bytes_current": 811403776, + "inactive_split_bytes_peak": 1283983872, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 24639438848, + "reserved_bytes_peak": 24639438848, + "segment_count_current": 477, + "segment_count_peak": 477, + "timestamp_utc": "2026-08-26T19:30:21.878867Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8926907904, + "inactive_split_bytes_current_delta": 8115504128, + "inactive_split_bytes_peak": 22443462656, + "inactive_split_bytes_peak_delta": 21159478784, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 133682954240, + "reserved_bytes_current_delta": 109043515392, + "reserved_bytes_peak": 133682954240, + "reserved_bytes_peak_delta": 109043515392, + "segment_count_current": 1126, + "segment_count_current_delta": 649, + "segment_count_peak": 1126, + "segment_count_peak_delta": 649, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T193044093901Z.pickle", + "timestamp_utc": "2026-08-26T19:30:44.200644Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T193044093901Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T193044093901Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..42b22e4de006ee9271f95f397e4eb7366505c8f8 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T193044093901Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8a75392c9a1b26c6b9cf58570ab6ed1f5b0a98c0e1533eeaa45784ce4b96089 +size 5169792 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T193044942837Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T193044942837Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c78857701e53041390a217294cea17284d8d33f1 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T193044942837Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69357679104, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145251749888, + "active_bytes_peak_delta": 14219808256, + "after": { + "active_bytes_current": 69357679104, + "active_bytes_peak": 145251749888, + "allocated_bytes_current": 69357679104, + "allocated_bytes_peak": 145251749888, + "device": 3, + "inactive_split_bytes_current": 8926907904, + "inactive_split_bytes_peak": 22443462656, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1138, + "segment_count_peak": 1138, + "timestamp_utc": "2026-08-26T19:30:44.942426Z", + "world_size": 8 + }, + "allocated_bytes_current": 69357679104, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145251749888, + "allocated_bytes_peak_delta": 14219808256, + "before": { + "active_bytes_current": 69357679104, + "active_bytes_peak": 131031941632, + "allocated_bytes_current": 69357679104, + "allocated_bytes_peak": 131031941632, + "device": 3, + "inactive_split_bytes_current": 8926907904, + "inactive_split_bytes_peak": 22443462656, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 133682954240, + "reserved_bytes_peak": 133682954240, + "segment_count_current": 1126, + "segment_count_peak": 1126, + "timestamp_utc": "2026-08-26T19:30:44.208818Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8926907904, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22443462656, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 148226703360, + "reserved_bytes_current_delta": 14543749120, + "reserved_bytes_peak": 148226703360, + "reserved_bytes_peak_delta": 14543749120, + "segment_count_current": 1138, + "segment_count_current_delta": 12, + "segment_count_peak": 1138, + "segment_count_peak_delta": 12, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T193044942837Z.pickle", + "timestamp_utc": "2026-08-26T19:30:45.028315Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T193044942837Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T193044942837Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0ceb1e935631a2cae9099271a29aa1e51111e8d0 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T193044942837Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f37b2874084c81e1c6078b2f4db5ee18551f6eca6f1bd9979f24568140977f0 +size 4397518 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T193045765121Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T193045765121Z.json new file mode 100644 index 0000000000000000000000000000000000000000..37e0109e528dad9dbc08e4cbb0a5c94e6477fe34 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T193045765121Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69357679104, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145253970432, + "active_bytes_peak_delta": 2220544, + "after": { + "active_bytes_current": 69357679104, + "active_bytes_peak": 145253970432, + "allocated_bytes_current": 69357679104, + "allocated_bytes_peak": 145253970432, + "device": 3, + "inactive_split_bytes_current": 8926907904, + "inactive_split_bytes_peak": 22443462656, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1138, + "segment_count_peak": 1138, + "timestamp_utc": "2026-08-26T19:30:45.764722Z", + "world_size": 8 + }, + "allocated_bytes_current": 69357679104, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145253970432, + "allocated_bytes_peak_delta": 2220544, + "before": { + "active_bytes_current": 69357679104, + "active_bytes_peak": 145251749888, + "allocated_bytes_current": 69357679104, + "allocated_bytes_peak": 145251749888, + "device": 3, + "inactive_split_bytes_current": 8926907904, + "inactive_split_bytes_peak": 22443462656, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1138, + "segment_count_peak": 1138, + "timestamp_utc": "2026-08-26T19:30:45.034843Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8926907904, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22443462656, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 148226703360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148226703360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1138, + "segment_count_current_delta": 0, + "segment_count_peak": 1138, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T193045765121Z.pickle", + "timestamp_utc": "2026-08-26T19:30:45.847590Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T193045765121Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T193045765121Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..20620868ae321e21b85eb56c8bb8deb353aead90 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T193045765121Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c727e4afd99b774b143e7f0a9d55ceb4144243b2adc664558fc578919c8289fb +size 4397416 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T193047892621Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T193047892621Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9a688d608d8b135f95098af88d0d4cafbb7f8017 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T193047892621Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69375640064, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 145253970432, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69375640064, + "active_bytes_peak": 145253970432, + "allocated_bytes_current": 69375640064, + "allocated_bytes_peak": 145253970432, + "device": 3, + "inactive_split_bytes_current": 11102567936, + "inactive_split_bytes_peak": 22443462656, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 156722266112, + "reserved_bytes_peak": 156722266112, + "segment_count_current": 1236, + "segment_count_peak": 1236, + "timestamp_utc": "2026-08-26T19:30:47.892130Z", + "world_size": 8 + }, + "allocated_bytes_current": 69375640064, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 145253970432, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69357679104, + "active_bytes_peak": 145253970432, + "allocated_bytes_current": 69357679104, + "allocated_bytes_peak": 145253970432, + "device": 3, + "inactive_split_bytes_current": 8926907904, + "inactive_split_bytes_peak": 22443462656, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1138, + "segment_count_peak": 1138, + "timestamp_utc": "2026-08-26T19:30:45.853887Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 11102567936, + "inactive_split_bytes_current_delta": 2175660032, + "inactive_split_bytes_peak": 22443462656, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 156722266112, + "reserved_bytes_current_delta": 8495562752, + "reserved_bytes_peak": 156722266112, + "reserved_bytes_peak_delta": 8495562752, + "segment_count_current": 1236, + "segment_count_current_delta": 98, + "segment_count_peak": 1236, + "segment_count_peak_delta": 98, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T193047892621Z.pickle", + "timestamp_utc": "2026-08-26T19:30:47.994313Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T193047892621Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T193047892621Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cad0dea4dc61677b3c01647da3766e6db1ac2d9c --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T193047892621Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37133411952d3209e78340c078cdbeaacfa07d0076bac23ae4f023460fcb8f2d +size 4506351 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T193048707026Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T193048707026Z.json new file mode 100644 index 0000000000000000000000000000000000000000..99e976ce3252e080f4cb7c9419b5fe6bcf67a853 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T193048707026Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69375640064, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145253970432, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69375640064, + "active_bytes_peak": 145253970432, + "allocated_bytes_current": 69375640064, + "allocated_bytes_peak": 145253970432, + "device": 3, + "inactive_split_bytes_current": 11102567936, + "inactive_split_bytes_peak": 22443462656, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 156722266112, + "reserved_bytes_peak": 156722266112, + "segment_count_current": 1236, + "segment_count_peak": 1236, + "timestamp_utc": "2026-08-26T19:30:48.706627Z", + "world_size": 8 + }, + "allocated_bytes_current": 69375640064, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145253970432, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69375640064, + "active_bytes_peak": 145253970432, + "allocated_bytes_current": 69375640064, + "allocated_bytes_peak": 145253970432, + "device": 3, + "inactive_split_bytes_current": 11102567936, + "inactive_split_bytes_peak": 22443462656, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 156722266112, + "reserved_bytes_peak": 156722266112, + "segment_count_current": 1236, + "segment_count_peak": 1236, + "timestamp_utc": "2026-08-26T19:30:48.001675Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 11102567936, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22443462656, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 156722266112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 156722266112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1236, + "segment_count_current_delta": 0, + "segment_count_peak": 1236, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T193048707026Z.pickle", + "timestamp_utc": "2026-08-26T19:30:48.756986Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T193048707026Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T193048707026Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ee1ec1d9b728a20effc98846741bd1e46a956070 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T193048707026Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d51ac2918dfa1f077e0b2adfece204cf01e715bd8e3aa6b665b84557c60d5967 +size 2403312 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T193049474994Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T193049474994Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1088c078fbcfd85c8557f6fe381b5e5ec0683806 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T193049474994Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69375640064, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145253970432, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69375640064, + "active_bytes_peak": 145253970432, + "allocated_bytes_current": 69375640064, + "allocated_bytes_peak": 145253970432, + "device": 3, + "inactive_split_bytes_current": 11102567936, + "inactive_split_bytes_peak": 22443462656, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 156722266112, + "reserved_bytes_peak": 156722266112, + "segment_count_current": 1236, + "segment_count_peak": 1236, + "timestamp_utc": "2026-08-26T19:30:49.474599Z", + "world_size": 8 + }, + "allocated_bytes_current": 69375640064, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145253970432, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69375640064, + "active_bytes_peak": 145253970432, + "allocated_bytes_current": 69375640064, + "allocated_bytes_peak": 145253970432, + "device": 3, + "inactive_split_bytes_current": 11102567936, + "inactive_split_bytes_peak": 22443462656, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 156722266112, + "reserved_bytes_peak": 156722266112, + "segment_count_current": 1236, + "segment_count_peak": 1236, + "timestamp_utc": "2026-08-26T19:30:48.759409Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 11102567936, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22443462656, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 156722266112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 156722266112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1236, + "segment_count_current_delta": 0, + "segment_count_peak": 1236, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T193049474994Z.pickle", + "timestamp_utc": "2026-08-26T19:30:49.522750Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T193049474994Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T193049474994Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..98f0f42adf7bf1f3155a2f585243826a22f84064 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T193049474994Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b9734810bf0c695ca4442abb58cd48c68211a8027890b2d129becf3b8cc2e2d +size 2403312 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T193044093891Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T193044093891Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3923fe2bcb750eb739598341a78171a6e8012c10 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T193044093891Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69359797248, + "active_bytes_current_delta": 45555914752, + "active_bytes_peak": 131032443392, + "active_bytes_peak_delta": 107215977984, + "after": { + "active_bytes_current": 69359797248, + "active_bytes_peak": 131032443392, + "allocated_bytes_current": 69359797248, + "allocated_bytes_peak": 131032443392, + "device": 4, + "inactive_split_bytes_current": 8941566976, + "inactive_split_bytes_peak": 22465937920, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 133699731456, + "reserved_bytes_peak": 133699731456, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:30:44.093382Z", + "world_size": 8 + }, + "allocated_bytes_current": 69359797248, + "allocated_bytes_current_delta": 45555914752, + "allocated_bytes_peak": 131032443392, + "allocated_bytes_peak_delta": 107215977984, + "before": { + "active_bytes_current": 23803882496, + "active_bytes_peak": 23816465408, + "allocated_bytes_current": 23803882496, + "allocated_bytes_peak": 23816465408, + "device": 4, + "inactive_split_bytes_current": 768447488, + "inactive_split_bytes_peak": 1283983872, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 24589107200, + "reserved_bytes_peak": 24589107200, + "segment_count_current": 473, + "segment_count_peak": 473, + "timestamp_utc": "2026-08-26T19:30:21.878622Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8941566976, + "inactive_split_bytes_current_delta": 8173119488, + "inactive_split_bytes_peak": 22465937920, + "inactive_split_bytes_peak_delta": 21181954048, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 133699731456, + "reserved_bytes_current_delta": 109110624256, + "reserved_bytes_peak": 133699731456, + "reserved_bytes_peak_delta": 109110624256, + "segment_count_current": 1123, + "segment_count_current_delta": 650, + "segment_count_peak": 1123, + "segment_count_peak_delta": 650, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T193044093891Z.pickle", + "timestamp_utc": "2026-08-26T19:30:44.201504Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T193044093891Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T193044093891Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..afa7a4b8f36f974a4cf4f78c0b6815b3fc874b3f --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T193044093891Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb9eab19161bbd589e1ede3b6f71b686eee2a7559e156ef1e781e381caf8df02 +size 4755788 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T193044942851Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T193044942851Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6174d7aa286a00e0efa0f9b1bcdb89c9b44da32c --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T193044942851Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69359797248, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145266951168, + "active_bytes_peak_delta": 14234507776, + "after": { + "active_bytes_current": 69359797248, + "active_bytes_peak": 145266951168, + "allocated_bytes_current": 69359797248, + "allocated_bytes_peak": 145266951168, + "device": 4, + "inactive_split_bytes_current": 8941566976, + "inactive_split_bytes_peak": 22465937920, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 148243480576, + "reserved_bytes_peak": 148243480576, + "segment_count_current": 1135, + "segment_count_peak": 1135, + "timestamp_utc": "2026-08-26T19:30:44.942445Z", + "world_size": 8 + }, + "allocated_bytes_current": 69359797248, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145266951168, + "allocated_bytes_peak_delta": 14234507776, + "before": { + "active_bytes_current": 69359797248, + "active_bytes_peak": 131032443392, + "allocated_bytes_current": 69359797248, + "allocated_bytes_peak": 131032443392, + "device": 4, + "inactive_split_bytes_current": 8941566976, + "inactive_split_bytes_peak": 22465937920, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 133699731456, + "reserved_bytes_peak": 133699731456, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:30:44.209567Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8941566976, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22465937920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 148243480576, + "reserved_bytes_current_delta": 14543749120, + "reserved_bytes_peak": 148243480576, + "reserved_bytes_peak_delta": 14543749120, + "segment_count_current": 1135, + "segment_count_current_delta": 12, + "segment_count_peak": 1135, + "segment_count_peak_delta": 12, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T193044942851Z.pickle", + "timestamp_utc": "2026-08-26T19:30:45.029278Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T193044942851Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T193044942851Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3e064d2d051bb59d0570da7f13d908fc3c615f53 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T193044942851Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a83d5153c2f0c0b9b55e6ac374638bcf1625d02f30e4e558f114e10fa0f3060 +size 3980709 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T193045765144Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T193045765144Z.json new file mode 100644 index 0000000000000000000000000000000000000000..548530597ef1b4fd6e25c8e9ed01c7fedaa18896 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T193045765144Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69359797248, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145270359040, + "active_bytes_peak_delta": 3407872, + "after": { + "active_bytes_current": 69359797248, + "active_bytes_peak": 145270359040, + "allocated_bytes_current": 69359797248, + "allocated_bytes_peak": 145270359040, + "device": 4, + "inactive_split_bytes_current": 8941566976, + "inactive_split_bytes_peak": 22465937920, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 148243480576, + "reserved_bytes_peak": 148243480576, + "segment_count_current": 1135, + "segment_count_peak": 1135, + "timestamp_utc": "2026-08-26T19:30:45.764747Z", + "world_size": 8 + }, + "allocated_bytes_current": 69359797248, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145270359040, + "allocated_bytes_peak_delta": 3407872, + "before": { + "active_bytes_current": 69359797248, + "active_bytes_peak": 145266951168, + "allocated_bytes_current": 69359797248, + "allocated_bytes_peak": 145266951168, + "device": 4, + "inactive_split_bytes_current": 8941566976, + "inactive_split_bytes_peak": 22465937920, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 148243480576, + "reserved_bytes_peak": 148243480576, + "segment_count_current": 1135, + "segment_count_peak": 1135, + "timestamp_utc": "2026-08-26T19:30:45.035801Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8941566976, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22465937920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 148243480576, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148243480576, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1135, + "segment_count_current_delta": 0, + "segment_count_peak": 1135, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T193045765144Z.pickle", + "timestamp_utc": "2026-08-26T19:30:45.848275Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T193045765144Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T193045765144Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1dd21dc541154c2bfa59668cf8679fa007875ea0 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T193045765144Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26ace5981b4e6d770f047578bfd4e9049bfbe00039238827a9690598dabc71b5 +size 3980641 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T193047892531Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T193047892531Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ed82448847cfa0520d594bcca8b93c5c97171454 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T193047892531Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69377758208, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 145270359040, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69377758208, + "active_bytes_peak": 145270359040, + "allocated_bytes_current": 69377758208, + "allocated_bytes_peak": 145270359040, + "device": 4, + "inactive_split_bytes_current": 11117227008, + "inactive_split_bytes_peak": 22465937920, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 156739043328, + "reserved_bytes_peak": 156739043328, + "segment_count_current": 1233, + "segment_count_peak": 1233, + "timestamp_utc": "2026-08-26T19:30:47.892084Z", + "world_size": 8 + }, + "allocated_bytes_current": 69377758208, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 145270359040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69359797248, + "active_bytes_peak": 145270359040, + "allocated_bytes_current": 69359797248, + "allocated_bytes_peak": 145270359040, + "device": 4, + "inactive_split_bytes_current": 8941566976, + "inactive_split_bytes_peak": 22465937920, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 148243480576, + "reserved_bytes_peak": 148243480576, + "segment_count_current": 1135, + "segment_count_peak": 1135, + "timestamp_utc": "2026-08-26T19:30:45.854519Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 11117227008, + "inactive_split_bytes_current_delta": 2175660032, + "inactive_split_bytes_peak": 22465937920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 156739043328, + "reserved_bytes_current_delta": 8495562752, + "reserved_bytes_peak": 156739043328, + "reserved_bytes_peak_delta": 8495562752, + "segment_count_current": 1233, + "segment_count_current_delta": 98, + "segment_count_peak": 1233, + "segment_count_peak_delta": 98, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T193047892531Z.pickle", + "timestamp_utc": "2026-08-26T19:30:47.993603Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T193047892531Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T193047892531Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..332a44a921e3e4d68142359845d3cc63c591a3e8 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T193047892531Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb09204d950590475230cf719eec99cdacf8610786f9e80bc43a7cf828abdac5 +size 4089661 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T193048707018Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T193048707018Z.json new file mode 100644 index 0000000000000000000000000000000000000000..90fad2335ad373c0b5bf9169708b975f40b7220d --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T193048707018Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69377758208, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145270359040, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69377758208, + "active_bytes_peak": 145270359040, + "allocated_bytes_current": 69377758208, + "allocated_bytes_peak": 145270359040, + "device": 4, + "inactive_split_bytes_current": 11117227008, + "inactive_split_bytes_peak": 22465937920, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 156739043328, + "reserved_bytes_peak": 156739043328, + "segment_count_current": 1233, + "segment_count_peak": 1233, + "timestamp_utc": "2026-08-26T19:30:48.706622Z", + "world_size": 8 + }, + "allocated_bytes_current": 69377758208, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145270359040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69377758208, + "active_bytes_peak": 145270359040, + "allocated_bytes_current": 69377758208, + "allocated_bytes_peak": 145270359040, + "device": 4, + "inactive_split_bytes_current": 11117227008, + "inactive_split_bytes_peak": 22465937920, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 156739043328, + "reserved_bytes_peak": 156739043328, + "segment_count_current": 1233, + "segment_count_peak": 1233, + "timestamp_utc": "2026-08-26T19:30:48.001176Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 11117227008, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22465937920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 156739043328, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 156739043328, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1233, + "segment_count_current_delta": 0, + "segment_count_peak": 1233, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T193048707018Z.pickle", + "timestamp_utc": "2026-08-26T19:30:48.756690Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T193048707018Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T193048707018Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7e37fede193264232e38659ac2a347c9f1e15d2a --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T193048707018Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87add0d5c86a6173b595970471c37663b050bafdeb0feb10ac4850576885b8f0 +size 2013742 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T193049475021Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T193049475021Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fc116e6490e920da4e2ea6a6f84adbb55125ec76 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T193049475021Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69377758208, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145270359040, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69377758208, + "active_bytes_peak": 145270359040, + "allocated_bytes_current": 69377758208, + "allocated_bytes_peak": 145270359040, + "device": 4, + "inactive_split_bytes_current": 11117227008, + "inactive_split_bytes_peak": 22465937920, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 156739043328, + "reserved_bytes_peak": 156739043328, + "segment_count_current": 1233, + "segment_count_peak": 1233, + "timestamp_utc": "2026-08-26T19:30:49.474634Z", + "world_size": 8 + }, + "allocated_bytes_current": 69377758208, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145270359040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69377758208, + "active_bytes_peak": 145270359040, + "allocated_bytes_current": 69377758208, + "allocated_bytes_peak": 145270359040, + "device": 4, + "inactive_split_bytes_current": 11117227008, + "inactive_split_bytes_peak": 22465937920, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 156739043328, + "reserved_bytes_peak": 156739043328, + "segment_count_current": 1233, + "segment_count_peak": 1233, + "timestamp_utc": "2026-08-26T19:30:48.759189Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 11117227008, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22465937920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 156739043328, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 156739043328, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1233, + "segment_count_current_delta": 0, + "segment_count_peak": 1233, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T193049475021Z.pickle", + "timestamp_utc": "2026-08-26T19:30:49.524302Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T193049475021Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T193049475021Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1463fa20d638ae2882fb1c3eee607f5dfb7126f2 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T193049475021Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68fced60250246ca13fa1294ed47f9ae78888f81b599b2e2222f7b8129e49eab +size 2013742 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T193044093904Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T193044093904Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0e408cac4e15c3aa6dec863b90e95ecbcc2116ae --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T193044093904Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69366880768, + "active_bytes_current_delta": 45554004480, + "active_bytes_peak": 131036776960, + "active_bytes_peak_delta": 107211317760, + "after": { + "active_bytes_current": 69366880768, + "active_bytes_peak": 131036776960, + "allocated_bytes_current": 69366880768, + "allocated_bytes_peak": 131036776960, + "device": 5, + "inactive_split_bytes_current": 8913511936, + "inactive_split_bytes_peak": 22434656256, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 133680857088, + "reserved_bytes_peak": 133680857088, + "segment_count_current": 1125, + "segment_count_peak": 1125, + "timestamp_utc": "2026-08-26T19:30:44.093389Z", + "world_size": 8 + }, + "allocated_bytes_current": 69366880768, + "allocated_bytes_current_delta": 45554004480, + "allocated_bytes_peak": 131036776960, + "allocated_bytes_peak_delta": 107211317760, + "before": { + "active_bytes_current": 23812876288, + "active_bytes_peak": 23825459200, + "allocated_bytes_current": 23812876288, + "allocated_bytes_peak": 23825459200, + "device": 5, + "inactive_split_bytes_current": 807688192, + "inactive_split_bytes_peak": 1283983872, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 24639438848, + "reserved_bytes_peak": 24639438848, + "segment_count_current": 477, + "segment_count_peak": 477, + "timestamp_utc": "2026-08-26T19:30:21.878830Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8913511936, + "inactive_split_bytes_current_delta": 8105823744, + "inactive_split_bytes_peak": 22434656256, + "inactive_split_bytes_peak_delta": 21150672384, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 133680857088, + "reserved_bytes_current_delta": 109041418240, + "reserved_bytes_peak": 133680857088, + "reserved_bytes_peak_delta": 109041418240, + "segment_count_current": 1125, + "segment_count_current_delta": 648, + "segment_count_peak": 1125, + "segment_count_peak_delta": 648, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T193044093904Z.pickle", + "timestamp_utc": "2026-08-26T19:30:44.207806Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T193044093904Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T193044093904Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e1ac4985a8fe8cfb6880d2272d12d8c1f07950b4 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T193044093904Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ad536f0de31b5e0ccb536e20ed466b37b5cf120c6f3ab1e25daf2ead751f611 +size 5161885 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T193044942871Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T193044942871Z.json new file mode 100644 index 0000000000000000000000000000000000000000..25c5f53c7716f0f57685979951104b0fe8d5bd01 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T193044942871Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69366880768, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145260196352, + "active_bytes_peak_delta": 14223419392, + "after": { + "active_bytes_current": 69366880768, + "active_bytes_peak": 145260196352, + "allocated_bytes_current": 69366880768, + "allocated_bytes_peak": 145260196352, + "device": 5, + "inactive_split_bytes_current": 8913511936, + "inactive_split_bytes_peak": 22434656256, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 148220411904, + "reserved_bytes_peak": 148220411904, + "segment_count_current": 1135, + "segment_count_peak": 1135, + "timestamp_utc": "2026-08-26T19:30:44.942469Z", + "world_size": 8 + }, + "allocated_bytes_current": 69366880768, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145260196352, + "allocated_bytes_peak_delta": 14223419392, + "before": { + "active_bytes_current": 69366880768, + "active_bytes_peak": 131036776960, + "allocated_bytes_current": 69366880768, + "allocated_bytes_peak": 131036776960, + "device": 5, + "inactive_split_bytes_current": 8913511936, + "inactive_split_bytes_peak": 22434656256, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 133680857088, + "reserved_bytes_peak": 133680857088, + "segment_count_current": 1125, + "segment_count_peak": 1125, + "timestamp_utc": "2026-08-26T19:30:44.216960Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8913511936, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22434656256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 148220411904, + "reserved_bytes_current_delta": 14539554816, + "reserved_bytes_peak": 148220411904, + "reserved_bytes_peak_delta": 14539554816, + "segment_count_current": 1135, + "segment_count_current_delta": 10, + "segment_count_peak": 1135, + "segment_count_peak_delta": 10, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T193044942871Z.pickle", + "timestamp_utc": "2026-08-26T19:30:45.031796Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T193044942871Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T193044942871Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9e114146b5ed560825da690b95f46e360230963f --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T193044942871Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3964ac97cccad31e1017190fb413a1f471468f19e3a098b8ea6ff50a93e7c775 +size 4390586 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193045765129Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193045765129Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a57c513bef443ba52cb44a34d6d428c660708e38 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193045765129Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69366880768, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145264652800, + "active_bytes_peak_delta": 4456448, + "after": { + "active_bytes_current": 69366880768, + "active_bytes_peak": 145264652800, + "allocated_bytes_current": 69366880768, + "allocated_bytes_peak": 145264652800, + "device": 5, + "inactive_split_bytes_current": 8913511936, + "inactive_split_bytes_peak": 22434656256, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 148220411904, + "reserved_bytes_peak": 148220411904, + "segment_count_current": 1135, + "segment_count_peak": 1135, + "timestamp_utc": "2026-08-26T19:30:45.764705Z", + "world_size": 8 + }, + "allocated_bytes_current": 69366880768, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145264652800, + "allocated_bytes_peak_delta": 4456448, + "before": { + "active_bytes_current": 69366880768, + "active_bytes_peak": 145260196352, + "allocated_bytes_current": 69366880768, + "allocated_bytes_peak": 145260196352, + "device": 5, + "inactive_split_bytes_current": 8913511936, + "inactive_split_bytes_peak": 22434656256, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 148220411904, + "reserved_bytes_peak": 148220411904, + "segment_count_current": 1135, + "segment_count_peak": 1135, + "timestamp_utc": "2026-08-26T19:30:45.039300Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8913511936, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22434656256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 148220411904, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148220411904, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1135, + "segment_count_current_delta": 0, + "segment_count_peak": 1135, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T193045765129Z.pickle", + "timestamp_utc": "2026-08-26T19:30:45.852015Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193045765129Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193045765129Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9456720334cfb8c6a0d4a2ddd614af52c09e4d00 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193045765129Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84ab8771591f25eb63ea52a30fdf2293730d9907cded5cc4f0f1243cf52f75dc +size 4390804 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193047892586Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193047892586Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a6f97f32e09c6dc1dfa66b105a6259032cecdadb --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193047892586Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69384841728, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 145264652800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69384841728, + "active_bytes_peak": 145264652800, + "allocated_bytes_current": 69384841728, + "allocated_bytes_peak": 145264652800, + "device": 5, + "inactive_split_bytes_current": 11089171968, + "inactive_split_bytes_peak": 22434656256, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 156718071808, + "reserved_bytes_peak": 156718071808, + "segment_count_current": 1234, + "segment_count_peak": 1234, + "timestamp_utc": "2026-08-26T19:30:47.892092Z", + "world_size": 8 + }, + "allocated_bytes_current": 69384841728, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 145264652800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69366880768, + "active_bytes_peak": 145264652800, + "allocated_bytes_current": 69366880768, + "allocated_bytes_peak": 145264652800, + "device": 5, + "inactive_split_bytes_current": 8913511936, + "inactive_split_bytes_peak": 22434656256, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 148220411904, + "reserved_bytes_peak": 148220411904, + "segment_count_current": 1135, + "segment_count_peak": 1135, + "timestamp_utc": "2026-08-26T19:30:45.859355Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 11089171968, + "inactive_split_bytes_current_delta": 2175660032, + "inactive_split_bytes_peak": 22434656256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 156718071808, + "reserved_bytes_current_delta": 8497659904, + "reserved_bytes_peak": 156718071808, + "reserved_bytes_peak_delta": 8497659904, + "segment_count_current": 1234, + "segment_count_current_delta": 99, + "segment_count_peak": 1234, + "segment_count_peak_delta": 99, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T193047892586Z.pickle", + "timestamp_utc": "2026-08-26T19:30:47.989857Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193047892586Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193047892586Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8024df10e9237d4142dca485a0e2c04115ee788f --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193047892586Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c539d174632ec0e29bcdac577a97e29b6c85fe97b85bb692eccd796901d2c6a6 +size 4504587 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193048707175Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193048707175Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ca08869c67ab2866e60cf1d73890172be65514c5 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193048707175Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69384841728, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145264652800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69384841728, + "active_bytes_peak": 145264652800, + "allocated_bytes_current": 69384841728, + "allocated_bytes_peak": 145264652800, + "device": 5, + "inactive_split_bytes_current": 11089171968, + "inactive_split_bytes_peak": 22434656256, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 156718071808, + "reserved_bytes_peak": 156718071808, + "segment_count_current": 1234, + "segment_count_peak": 1234, + "timestamp_utc": "2026-08-26T19:30:48.706778Z", + "world_size": 8 + }, + "allocated_bytes_current": 69384841728, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145264652800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69384841728, + "active_bytes_peak": 145264652800, + "allocated_bytes_current": 69384841728, + "allocated_bytes_peak": 145264652800, + "device": 5, + "inactive_split_bytes_current": 11089171968, + "inactive_split_bytes_peak": 22434656256, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 156718071808, + "reserved_bytes_peak": 156718071808, + "segment_count_current": 1234, + "segment_count_peak": 1234, + "timestamp_utc": "2026-08-26T19:30:47.997664Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 11089171968, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22434656256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 156718071808, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 156718071808, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1234, + "segment_count_current_delta": 0, + "segment_count_peak": 1234, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T193048707175Z.pickle", + "timestamp_utc": "2026-08-26T19:30:48.769892Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193048707175Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193048707175Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..277b61bc1eeeaaa200c64edbdf6d9bb03118ddf0 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193048707175Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c96195caf5d27359ed921df053130cee465b8a8570fbe9aabaa1a2a537d59270 +size 2401130 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T193049475017Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T193049475017Z.json new file mode 100644 index 0000000000000000000000000000000000000000..210e64e44dc8b572b6627cec94b0a988bf9c97dd --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T193049475017Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69384841728, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145264652800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69384841728, + "active_bytes_peak": 145264652800, + "allocated_bytes_current": 69384841728, + "allocated_bytes_peak": 145264652800, + "device": 5, + "inactive_split_bytes_current": 11089171968, + "inactive_split_bytes_peak": 22434656256, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 156718071808, + "reserved_bytes_peak": 156718071808, + "segment_count_current": 1234, + "segment_count_peak": 1234, + "timestamp_utc": "2026-08-26T19:30:49.474618Z", + "world_size": 8 + }, + "allocated_bytes_current": 69384841728, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145264652800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69384841728, + "active_bytes_peak": 145264652800, + "allocated_bytes_current": 69384841728, + "allocated_bytes_peak": 145264652800, + "device": 5, + "inactive_split_bytes_current": 11089171968, + "inactive_split_bytes_peak": 22434656256, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 156718071808, + "reserved_bytes_peak": 156718071808, + "segment_count_current": 1234, + "segment_count_peak": 1234, + "timestamp_utc": "2026-08-26T19:30:48.772336Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 11089171968, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22434656256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 156718071808, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 156718071808, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1234, + "segment_count_current_delta": 0, + "segment_count_peak": 1234, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T193049475017Z.pickle", + "timestamp_utc": "2026-08-26T19:30:49.526800Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T193049475017Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T193049475017Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e1342d47047165defbe50f345e7c4b0f09422953 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T193049475017Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8cd9f50ed67a573e8fc72e1bbccdd39a3b7509471579b10685092e06a0cdca6 +size 2401096 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193044093949Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193044093949Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5a382ba672f983752fc8bba6ea8d59586b06fb77 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193044093949Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69357192192, + "active_bytes_current_delta": 45562908672, + "active_bytes_peak": 131019717632, + "active_bytes_peak_delta": 107212851200, + "after": { + "active_bytes_current": 69357192192, + "active_bytes_peak": 131019717632, + "allocated_bytes_current": 69357192192, + "allocated_bytes_peak": 131019717632, + "device": 6, + "inactive_split_bytes_current": 8916909056, + "inactive_split_bytes_peak": 22444003840, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 134496649216, + "reserved_bytes_peak": 134496649216, + "segment_count_current": 1122, + "segment_count_peak": 1122, + "timestamp_utc": "2026-08-26T19:30:44.093434Z", + "world_size": 8 + }, + "allocated_bytes_current": 69357192192, + "allocated_bytes_current_delta": 45562908672, + "allocated_bytes_peak": 131019717632, + "allocated_bytes_peak_delta": 107212851200, + "before": { + "active_bytes_current": 23794283520, + "active_bytes_peak": 23806866432, + "allocated_bytes_current": 23794283520, + "allocated_bytes_peak": 23806866432, + "device": 6, + "inactive_split_bytes_current": 815795200, + "inactive_split_bytes_peak": 1283983872, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 24626855936, + "reserved_bytes_peak": 24626855936, + "segment_count_current": 471, + "segment_count_peak": 471, + "timestamp_utc": "2026-08-26T19:30:21.878894Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8916909056, + "inactive_split_bytes_current_delta": 8101113856, + "inactive_split_bytes_peak": 22444003840, + "inactive_split_bytes_peak_delta": 21160019968, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 134496649216, + "reserved_bytes_current_delta": 109869793280, + "reserved_bytes_peak": 134496649216, + "reserved_bytes_peak_delta": 109869793280, + "segment_count_current": 1122, + "segment_count_current_delta": 651, + "segment_count_peak": 1122, + "segment_count_peak_delta": 651, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T193044093949Z.pickle", + "timestamp_utc": "2026-08-26T19:30:44.203570Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193044093949Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193044093949Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..333509e835bad50d24538ca5c3d353bddf6acccf --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193044093949Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da52eded4285a478fb88a353d2d5a913b47aa03c8b78ea78a56c75bee1f45c60 +size 4752079 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193044942868Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193044942868Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a6ac31f687037e5b7e4d7a7b579fa124feed1239 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193044942868Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69357192192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145253341184, + "active_bytes_peak_delta": 14233623552, + "after": { + "active_bytes_current": 69357192192, + "active_bytes_peak": 145253341184, + "allocated_bytes_current": 69357192192, + "allocated_bytes_peak": 145253341184, + "device": 6, + "inactive_split_bytes_current": 8916909056, + "inactive_split_bytes_peak": 22444003840, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 147394134016, + "reserved_bytes_peak": 147394134016, + "segment_count_current": 1134, + "segment_count_peak": 1134, + "timestamp_utc": "2026-08-26T19:30:44.942454Z", + "world_size": 8 + }, + "allocated_bytes_current": 69357192192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145253341184, + "allocated_bytes_peak_delta": 14233623552, + "before": { + "active_bytes_current": 69357192192, + "active_bytes_peak": 131019717632, + "allocated_bytes_current": 69357192192, + "allocated_bytes_peak": 131019717632, + "device": 6, + "inactive_split_bytes_current": 8916909056, + "inactive_split_bytes_peak": 22444003840, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 134496649216, + "reserved_bytes_peak": 134496649216, + "segment_count_current": 1122, + "segment_count_peak": 1122, + "timestamp_utc": "2026-08-26T19:30:44.211901Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8916909056, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22444003840, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 147394134016, + "reserved_bytes_current_delta": 12897484800, + "reserved_bytes_peak": 147394134016, + "reserved_bytes_peak_delta": 12897484800, + "segment_count_current": 1134, + "segment_count_current_delta": 12, + "segment_count_peak": 1134, + "segment_count_peak_delta": 12, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T193044942868Z.pickle", + "timestamp_utc": "2026-08-26T19:30:45.027993Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193044942868Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193044942868Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ee37590dc1cd520a2ac7aede90bf69caebca26e5 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193044942868Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b6ac34c3b62592b22e4c81ca5a0ed4316e3ba5683f582cba0c20f2be10a5fc2 +size 3975863 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193045765129Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193045765129Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d1a822fdf65f217eee8b1b50bfb3d2c361334463 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193045765129Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69357192192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145253341184, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69357192192, + "active_bytes_peak": 145253341184, + "allocated_bytes_current": 69357192192, + "allocated_bytes_peak": 145253341184, + "device": 6, + "inactive_split_bytes_current": 8916909056, + "inactive_split_bytes_peak": 22444003840, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 147394134016, + "reserved_bytes_peak": 147394134016, + "segment_count_current": 1134, + "segment_count_peak": 1134, + "timestamp_utc": "2026-08-26T19:30:45.764732Z", + "world_size": 8 + }, + "allocated_bytes_current": 69357192192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145253341184, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69357192192, + "active_bytes_peak": 145253341184, + "allocated_bytes_current": 69357192192, + "allocated_bytes_peak": 145253341184, + "device": 6, + "inactive_split_bytes_current": 8916909056, + "inactive_split_bytes_peak": 22444003840, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 147394134016, + "reserved_bytes_peak": 147394134016, + "segment_count_current": 1134, + "segment_count_peak": 1134, + "timestamp_utc": "2026-08-26T19:30:45.034608Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8916909056, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22444003840, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 147394134016, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 147394134016, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1134, + "segment_count_current_delta": 0, + "segment_count_peak": 1134, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T193045765129Z.pickle", + "timestamp_utc": "2026-08-26T19:30:45.847478Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193045765129Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193045765129Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..930d2c20814d522d40b38be09cb117d2efb0eebb --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193045765129Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbec38eb8fca94e09d05153ee36f8cab46eb0ffa6863ca32b1b220f454a0d708 +size 3975726 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193047892540Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193047892540Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a6d4b3e9722ffb67289ba5c38e2c2ef8104745ec --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193047892540Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69375153152, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 145253341184, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69375153152, + "active_bytes_peak": 145253341184, + "allocated_bytes_current": 69375153152, + "allocated_bytes_peak": 145253341184, + "device": 6, + "inactive_split_bytes_current": 11092569088, + "inactive_split_bytes_peak": 22444003840, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 156711780352, + "reserved_bytes_peak": 156711780352, + "segment_count_current": 1231, + "segment_count_peak": 1231, + "timestamp_utc": "2026-08-26T19:30:47.892064Z", + "world_size": 8 + }, + "allocated_bytes_current": 69375153152, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 145253341184, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69357192192, + "active_bytes_peak": 145253341184, + "allocated_bytes_current": 69357192192, + "allocated_bytes_peak": 145253341184, + "device": 6, + "inactive_split_bytes_current": 8916909056, + "inactive_split_bytes_peak": 22444003840, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 147394134016, + "reserved_bytes_peak": 147394134016, + "segment_count_current": 1134, + "segment_count_peak": 1134, + "timestamp_utc": "2026-08-26T19:30:45.853764Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 11092569088, + "inactive_split_bytes_current_delta": 2175660032, + "inactive_split_bytes_peak": 22444003840, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 156711780352, + "reserved_bytes_current_delta": 9317646336, + "reserved_bytes_peak": 156711780352, + "reserved_bytes_peak_delta": 9317646336, + "segment_count_current": 1231, + "segment_count_current_delta": 97, + "segment_count_peak": 1231, + "segment_count_peak_delta": 97, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T193047892540Z.pickle", + "timestamp_utc": "2026-08-26T19:30:47.990519Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193047892540Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193047892540Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..011981a0e2be4e0bb576eee7c577dc531f45f573 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193047892540Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:570bf69c3e5fa64b9c5f3d8afcd8e1e7b6a45eca47d2d7dcbf053e5e6f982e62 +size 4084371 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193048707091Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193048707091Z.json new file mode 100644 index 0000000000000000000000000000000000000000..02ac262299bc21c4cae7d0c4069d135e2460ab9b --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193048707091Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69375153152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145253341184, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69375153152, + "active_bytes_peak": 145253341184, + "allocated_bytes_current": 69375153152, + "allocated_bytes_peak": 145253341184, + "device": 6, + "inactive_split_bytes_current": 11092569088, + "inactive_split_bytes_peak": 22444003840, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 156711780352, + "reserved_bytes_peak": 156711780352, + "segment_count_current": 1231, + "segment_count_peak": 1231, + "timestamp_utc": "2026-08-26T19:30:48.706624Z", + "world_size": 8 + }, + "allocated_bytes_current": 69375153152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145253341184, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69375153152, + "active_bytes_peak": 145253341184, + "allocated_bytes_current": 69375153152, + "allocated_bytes_peak": 145253341184, + "device": 6, + "inactive_split_bytes_current": 11092569088, + "inactive_split_bytes_peak": 22444003840, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 156711780352, + "reserved_bytes_peak": 156711780352, + "segment_count_current": 1231, + "segment_count_peak": 1231, + "timestamp_utc": "2026-08-26T19:30:47.997734Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 11092569088, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22444003840, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 156711780352, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 156711780352, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1231, + "segment_count_current_delta": 0, + "segment_count_peak": 1231, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T193048707091Z.pickle", + "timestamp_utc": "2026-08-26T19:30:48.757817Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193048707091Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193048707091Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cee86530517f82c380c543209add431d01488e37 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193048707091Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5872ca97258ea643f06b8788f93a4f40d121fea1316249b199919e381d242c1c +size 2011496 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T193049475000Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T193049475000Z.json new file mode 100644 index 0000000000000000000000000000000000000000..da14b733a092a3d0c172d5a375344534cc8719b9 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T193049475000Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69375153152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145253341184, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69375153152, + "active_bytes_peak": 145253341184, + "allocated_bytes_current": 69375153152, + "allocated_bytes_peak": 145253341184, + "device": 6, + "inactive_split_bytes_current": 11092569088, + "inactive_split_bytes_peak": 22444003840, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 156711780352, + "reserved_bytes_peak": 156711780352, + "segment_count_current": 1231, + "segment_count_peak": 1231, + "timestamp_utc": "2026-08-26T19:30:49.474603Z", + "world_size": 8 + }, + "allocated_bytes_current": 69375153152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145253341184, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69375153152, + "active_bytes_peak": 145253341184, + "allocated_bytes_current": 69375153152, + "allocated_bytes_peak": 145253341184, + "device": 6, + "inactive_split_bytes_current": 11092569088, + "inactive_split_bytes_peak": 22444003840, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 156711780352, + "reserved_bytes_peak": 156711780352, + "segment_count_current": 1231, + "segment_count_peak": 1231, + "timestamp_utc": "2026-08-26T19:30:48.760358Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 11092569088, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22444003840, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 156711780352, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 156711780352, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1231, + "segment_count_current_delta": 0, + "segment_count_peak": 1231, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T193049475000Z.pickle", + "timestamp_utc": "2026-08-26T19:30:49.522667Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T193049475000Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T193049475000Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e7f397a406a571e96d12d5f51718d05b8619e8bd --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T193049475000Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e32b6320864a697684df037b0faf6a5ff957ed3af61ed2d8e7b25c8e73db4991 +size 2011496 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193044093906Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193044093906Z.json new file mode 100644 index 0000000000000000000000000000000000000000..72ae6e77f9d1b17f9ac88d8dffa06ba7fedd294c --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193044093906Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69360229888, + "active_bytes_current_delta": 45556949504, + "active_bytes_peak": 131036168704, + "active_bytes_peak_delta": 107220305408, + "after": { + "active_bytes_current": 69360229888, + "active_bytes_peak": 131036168704, + "allocated_bytes_current": 69360229888, + "allocated_bytes_peak": 131036168704, + "device": 7, + "inactive_split_bytes_current": 8926454272, + "inactive_split_bytes_peak": 22444142592, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 133687148544, + "reserved_bytes_peak": 133687148544, + "segment_count_current": 1127, + "segment_count_peak": 1127, + "timestamp_utc": "2026-08-26T19:30:44.093384Z", + "world_size": 8 + }, + "allocated_bytes_current": 69360229888, + "allocated_bytes_current_delta": 45556949504, + "allocated_bytes_peak": 131036168704, + "allocated_bytes_peak_delta": 107220305408, + "before": { + "active_bytes_current": 23803280384, + "active_bytes_peak": 23815863296, + "allocated_bytes_current": 23803280384, + "allocated_bytes_peak": 23815863296, + "device": 7, + "inactive_split_bytes_current": 890684416, + "inactive_split_bytes_peak": 1283983872, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 24712839168, + "reserved_bytes_peak": 24712839168, + "segment_count_current": 477, + "segment_count_peak": 477, + "timestamp_utc": "2026-08-26T19:30:21.878846Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8926454272, + "inactive_split_bytes_current_delta": 8035769856, + "inactive_split_bytes_peak": 22444142592, + "inactive_split_bytes_peak_delta": 21160158720, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 133687148544, + "reserved_bytes_current_delta": 108974309376, + "reserved_bytes_peak": 133687148544, + "reserved_bytes_peak_delta": 108974309376, + "segment_count_current": 1127, + "segment_count_current_delta": 650, + "segment_count_peak": 1127, + "segment_count_peak_delta": 650, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T193044093906Z.pickle", + "timestamp_utc": "2026-08-26T19:30:44.211452Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193044093906Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193044093906Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..14a289499116927b2138b867a212491325afc2c8 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193044093906Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c73b2ffb49bb1a326fe0064541df56c94310b3b219adc459f8190c2b9b4c4ca +size 4742714 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193044942869Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193044942869Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bbfce9c0733ff546c71b4a6abd480fa8ec34a802 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193044942869Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69360229888, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145248306176, + "active_bytes_peak_delta": 14212137472, + "after": { + "active_bytes_current": 69360229888, + "active_bytes_peak": 145248306176, + "allocated_bytes_current": 69360229888, + "allocated_bytes_peak": 145248306176, + "device": 7, + "inactive_split_bytes_current": 8926454272, + "inactive_split_bytes_peak": 22444142592, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:30:44.942458Z", + "world_size": 8 + }, + "allocated_bytes_current": 69360229888, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145248306176, + "allocated_bytes_peak_delta": 14212137472, + "before": { + "active_bytes_current": 69360229888, + "active_bytes_peak": 131036168704, + "allocated_bytes_current": 69360229888, + "allocated_bytes_peak": 131036168704, + "device": 7, + "inactive_split_bytes_current": 8926454272, + "inactive_split_bytes_peak": 22444142592, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 133687148544, + "reserved_bytes_peak": 133687148544, + "segment_count_current": 1127, + "segment_count_peak": 1127, + "timestamp_utc": "2026-08-26T19:30:44.220513Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8926454272, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22444142592, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 148226703360, + "reserved_bytes_current_delta": 14539554816, + "reserved_bytes_peak": 148226703360, + "reserved_bytes_peak_delta": 14539554816, + "segment_count_current": 1137, + "segment_count_current_delta": 10, + "segment_count_peak": 1137, + "segment_count_peak_delta": 10, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T193044942869Z.pickle", + "timestamp_utc": "2026-08-26T19:30:45.032245Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193044942869Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193044942869Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3fb73105d343f2f0aa46ff974fa15fc96f3474c6 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193044942869Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39dfa6ab4e52180bd8ef7d42100dfbcc50a1513a1701641af1432bdf47eb9304 +size 3971548 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193045765218Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193045765218Z.json new file mode 100644 index 0000000000000000000000000000000000000000..97b08059a9240a580bdb4831d29866602a6e4ca7 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193045765218Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69360229888, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145248306176, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69360229888, + "active_bytes_peak": 145248306176, + "allocated_bytes_current": 69360229888, + "allocated_bytes_peak": 145248306176, + "device": 7, + "inactive_split_bytes_current": 8926454272, + "inactive_split_bytes_peak": 22444142592, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:30:45.764810Z", + "world_size": 8 + }, + "allocated_bytes_current": 69360229888, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145248306176, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69360229888, + "active_bytes_peak": 145248306176, + "allocated_bytes_current": 69360229888, + "allocated_bytes_peak": 145248306176, + "device": 7, + "inactive_split_bytes_current": 8926454272, + "inactive_split_bytes_peak": 22444142592, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:30:45.039686Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8926454272, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22444142592, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 148226703360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148226703360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1137, + "segment_count_current_delta": 0, + "segment_count_peak": 1137, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T193045765218Z.pickle", + "timestamp_utc": "2026-08-26T19:30:45.852168Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193045765218Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193045765218Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..28b8b912da9be23a9085cb2963e01388dd5328bf --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193045765218Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33ed9532716196efd712c1d1ca83623b017011a317031b24c0258182c125fe65 +size 3971905 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193047892609Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193047892609Z.json new file mode 100644 index 0000000000000000000000000000000000000000..71f22e7c60969338d1bae6c5cc4b96538839a41b --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193047892609Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69378190848, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 145248306176, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69378190848, + "active_bytes_peak": 145248306176, + "allocated_bytes_current": 69378190848, + "allocated_bytes_peak": 145248306176, + "device": 7, + "inactive_split_bytes_current": 11102114304, + "inactive_split_bytes_peak": 22444142592, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 156724363264, + "reserved_bytes_peak": 156724363264, + "segment_count_current": 1236, + "segment_count_peak": 1236, + "timestamp_utc": "2026-08-26T19:30:47.892140Z", + "world_size": 8 + }, + "allocated_bytes_current": 69378190848, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 145248306176, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69360229888, + "active_bytes_peak": 145248306176, + "allocated_bytes_current": 69360229888, + "allocated_bytes_peak": 145248306176, + "device": 7, + "inactive_split_bytes_current": 8926454272, + "inactive_split_bytes_peak": 22444142592, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:30:45.859644Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 11102114304, + "inactive_split_bytes_current_delta": 2175660032, + "inactive_split_bytes_peak": 22444142592, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 156724363264, + "reserved_bytes_current_delta": 8497659904, + "reserved_bytes_peak": 156724363264, + "reserved_bytes_peak_delta": 8497659904, + "segment_count_current": 1236, + "segment_count_current_delta": 99, + "segment_count_peak": 1236, + "segment_count_peak_delta": 99, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T193047892609Z.pickle", + "timestamp_utc": "2026-08-26T19:30:47.989317Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193047892609Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193047892609Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cd20c148b4c8e08da531eb07d9c86bca4d939d64 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193047892609Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb848e2d1e3eafc1c0b739c07fdf384ae2a22691d9f8f87bfd74a6c7278c78ce +size 4080490 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193048707078Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193048707078Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0eac5ead74f3f1a07fdf24f25c22d986d146c367 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193048707078Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69378190848, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145248306176, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69378190848, + "active_bytes_peak": 145248306176, + "allocated_bytes_current": 69378190848, + "allocated_bytes_peak": 145248306176, + "device": 7, + "inactive_split_bytes_current": 11102114304, + "inactive_split_bytes_peak": 22444142592, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 156724363264, + "reserved_bytes_peak": 156724363264, + "segment_count_current": 1236, + "segment_count_peak": 1236, + "timestamp_utc": "2026-08-26T19:30:48.706662Z", + "world_size": 8 + }, + "allocated_bytes_current": 69378190848, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145248306176, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69378190848, + "active_bytes_peak": 145248306176, + "allocated_bytes_current": 69378190848, + "allocated_bytes_peak": 145248306176, + "device": 7, + "inactive_split_bytes_current": 11102114304, + "inactive_split_bytes_peak": 22444142592, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 156724363264, + "reserved_bytes_peak": 156724363264, + "segment_count_current": 1236, + "segment_count_peak": 1236, + "timestamp_utc": "2026-08-26T19:30:47.997008Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 11102114304, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22444142592, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 156724363264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 156724363264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1236, + "segment_count_current_delta": 0, + "segment_count_peak": 1236, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T193048707078Z.pickle", + "timestamp_utc": "2026-08-26T19:30:48.760297Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193048707078Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193048707078Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0b6a78772d5a3cedd53ca234605a9e24e9839802 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193048707078Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01db64e0cbaf1f1a47d5bc6ec270cb438a67cba1a2f2efee4029ceab92b5d6cf +size 2004660 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T193049475049Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T193049475049Z.json new file mode 100644 index 0000000000000000000000000000000000000000..71c7027326145139941095e2bd7cd35aedb7cd6c --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T193049475049Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69378190848, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145248306176, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69378190848, + "active_bytes_peak": 145248306176, + "allocated_bytes_current": 69378190848, + "allocated_bytes_peak": 145248306176, + "device": 7, + "inactive_split_bytes_current": 11102114304, + "inactive_split_bytes_peak": 22444142592, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 156724363264, + "reserved_bytes_peak": 156724363264, + "segment_count_current": 1236, + "segment_count_peak": 1236, + "timestamp_utc": "2026-08-26T19:30:49.474644Z", + "world_size": 8 + }, + "allocated_bytes_current": 69378190848, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145248306176, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69378190848, + "active_bytes_peak": 145248306176, + "allocated_bytes_current": 69378190848, + "allocated_bytes_peak": 145248306176, + "device": 7, + "inactive_split_bytes_current": 11102114304, + "inactive_split_bytes_peak": 22444142592, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 156724363264, + "reserved_bytes_peak": 156724363264, + "segment_count_current": 1236, + "segment_count_peak": 1236, + "timestamp_utc": "2026-08-26T19:30:48.762797Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 11102114304, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22444142592, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 156724363264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 156724363264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1236, + "segment_count_current_delta": 0, + "segment_count_peak": 1236, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T193049475049Z.pickle", + "timestamp_utc": "2026-08-26T19:30:49.525760Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T193049475049Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T193049475049Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c7fa22b1ce8a891d978e6ef6abc9c9120499e5a9 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T193049475049Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5131d125f1f99c1aa3484f32292808026353c8de06687538a5ca52fbe4051641 +size 2004660 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/run.env b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..eebde1aef2f1a8af7fc0f0e2f4043f0ae7b02240 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/run.env @@ -0,0 +1,61 @@ +backend=moonep +config=DSV2_LITE_L24 +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=moonep_backend_better_memory_dump_support +commit=453b84c7 +num_layers=24 +hidden=2048 +ffn_hidden=10944 +moe_ffn_hidden=1408 +num_experts=64 +topk=6 +experts_per_rank=8 +expert_params=8650752 +heads=16 +kv_groups=0 +kv_channels=128 +qk_layernorm=1 +shared_ffn=2816 +shared_expert_overlap=0 +mla=1 +dense_first_layers=1 +moe_layers=23 +rotary_base=10000 +rotary_scaling=40 +qkv_bias=0 +router_balancing=seq_aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=49152 +mean_tokens_per_expert=6144 +slot_policy=matched +slots_per_rank=8 +capacity_mode=none +capacity=none +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=6 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/train.log b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..ffcc4e24afe476422bdee4d5788eae7ce7a4d528 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/train.log @@ -0,0 +1,1423 @@ +============================================================== + DSV2_LITE_L24 / moonep (slots/rank=8 policy=matched) + E=64 K=6 H'=1408 m=8 + L=24 seq=8192 gbs=8 (1 microbatches) + cap_mode=none cap=none cg=1 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 453b84c7 +============================================================== +W0826 19:29:53.628000 679520 torch/distributed/run.py:766] +W0826 19:29:53.628000 679520 torch/distributed/run.py:766] ***************************************** +W0826 19:29:53.628000 679520 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 19:29:53.628000 679520 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 24 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 10944 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. False + moe_enable_moonep ............................... True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1408 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_moonep_num_sms .............................. None + moe_num_echo_experts ............................ None + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 6 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 2816 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 0.707 + mscale_all_dim .................................. 0.707 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 24 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 40.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 6 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 1[Gloo] Rank is connected to 47 is connected to 3 peer ranks. 7 is connected to Expected number of connected peer ranks is : peer ranks. 77Expected number of connected peer ranks is : peer ranks. +7Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 + +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 0312 is connected to is connected to is connected to is connected to 7777 peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7777 + + + +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank2]:[W826 19:30:12.224228885 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 19:30:12.224287948 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 19:30:12.224323359 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 19:30:12.224368446 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 19:30:12.224375456 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 19:30:12.225510030 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 19:30:12.226084834 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 19:30:12.397911037 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.625 seconds +time to initialize megatron (seconds): 6.963 +[after megatron is initialized] datetime: 2026-08-26 19:30:18 +building GPT model ... +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1531: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1531: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1531: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1531: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1531: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1531: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1531: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1531: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 2596386816 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (48503296 elements, 48503296 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.23.self_attention.linear_kv_down_proj.weight + module.decoder.layers.23.self_attention.linear_q_proj.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.23.input_layernorm.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight +Params for bucket 3 (40901632 elements, 40901632 padded size): + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.input_layernorm.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.21.self_attention.linear_kv_down_proj.weight + module.decoder.layers.21.self_attention.linear_q_proj.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.self_attention.linear_q_proj.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.weight +Params for bucket 4 (41163264 elements, 41163264 padded size): + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.21.input_layernorm.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.20.self_attention.linear_kv_down_proj.weight + module.decoder.layers.20.self_attention.linear_q_proj.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.20.input_layernorm.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight +Params for bucket 5 (42734080 elements, 42734080 padded size): + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.self_attention.linear_q_proj.weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.19.input_layernorm.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.self_attention.linear_kv_down_proj.weight +Params for bucket 6 (40901632 elements, 40901632 padded size): + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.18.input_layernorm.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.self_attention.linear_kv_down_proj.weight + module.decoder.layers.17.self_attention.linear_q_proj.weight +Params for bucket 7 (41163264 elements, 41163264 padded size): + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.weight + module.decoder.layers.16.self_attention.linear_q_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.16.input_layernorm.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.input_layernorm.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.16.self_attention.linear_kv_down_proj.weight +Params for bucket 8 (42734080 elements, 42734080 padded size): + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.15.self_attention.linear_q_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.15.input_layernorm.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.weight +Params for bucket 9 (40901632 elements, 40901632 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_q_proj.weight + module.decoder.layers.14.input_layernorm.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.13.self_attention.linear_kv_down_proj.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight +Params for bucket 10 (41163264 elements, 41163264 padded size): + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.12.self_attention.linear_kv_down_proj.weight + module.decoder.layers.12.self_attention.linear_q_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.input_layernorm.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.input_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.weight +Params for bucket 11 (42734080 elements, 42734080 padded size): + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_kv_down_proj.weight + module.decoder.layers.11.self_attention.linear_q_proj.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.input_layernorm.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight +Params for bucket 12 (40901632 elements, 40901632 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.weight + module.decoder.layers.9.self_attention.linear_q_proj.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.10.self_attention.linear_q_proj.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.input_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.9.self_attention.linear_kv_down_proj.weight +Params for bucket 13 (41163264 elements, 41163264 padded size): + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.input_layernorm.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.8.self_attention.linear_kv_down_proj.weight + module.decoder.layers.8.self_attention.linear_q_proj.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.input_layernorm.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight +Params for bucket 14 (42734080 elements, 42734080 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_kv_down_proj.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.input_layernorm.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight +Params for bucket 15 (40901632 elements, 40901632 padded size): + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.self_attention.linear_q_proj.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.input_layernorm.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.5.self_attention.linear_kv_down_proj.weight + module.decoder.layers.5.self_attention.linear_q_proj.weight +Params for bucket 16 (41163264 elements, 41163264 padded size): + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.4.self_attention.linear_kv_down_proj.weight + module.decoder.layers.4.self_attention.linear_q_proj.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.input_layernorm.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.input_layernorm.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.weight +Params for bucket 17 (42734080 elements, 42734080 padded size): + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_kv_down_proj.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_q_proj.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.input_layernorm.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight +Params for bucket 18 (40901632 elements, 40901632 padded size): + module.decoder.layers.2.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_q_proj.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.self_attention.linear_q_proj.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.input_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight +Params for bucket 19 (71436288 elements, 71436288 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.linear_fc1.weight + module.decoder.layers.0.mlp.linear_fc2.weight + module.decoder.layers.1.input_layernorm.weight +Params for bucket 20 (116789760 elements, 116789760 padded size): + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.self_attention.linear_q_proj.weight + module.decoder.layers.0.input_layernorm.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 +Params for bucket 3 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 +Params for bucket 4 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 +Params for bucket 7 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 +Params for bucket 8 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 +Params for bucket 11 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 +Params for bucket 12 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 +Params for bucket 15 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 +Params for bucket 16 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 +Params for bucket 19 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 +Params for bucket 20 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 23 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 +Params for bucket 24 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 25 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 +Params for bucket 28 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 +Params for bucket 29 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 +Params for bucket 32 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 33 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 +Params for bucket 36 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 37 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 40 (17301504 elements, 17301504 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 19:30:21 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 48 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(48, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 2c35baf848cdd0d293890dd0e79624da-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 19:30:21 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3578.18, 3589.49) + train/valid/test-data-iterators-setup ..........: (64.30, 69.40) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 19:30:21 +INFO:moonep.api:MoonEP comm buffer sizes: + config : R=8, S=8192, H=2048, E=64, K=6, NvS=51184, NvS_padded=51200 + sm_count : num_sms=32, num_sms_dedup=148 + hidden_buffer_size : 200.00 MiB + meta_buffer_size : 2.00 MiB + local_temp_buffer_size : 2.01 MiB + total_size : 204.01 MiB + [2026-08-26 19:30:44] iteration 1/ 6 | consumed samples: 8 | elapsed time per iteration (ms): 22215.3 | throughput per GPU (TFLOP/s/GPU): 5.7 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092206E+01 | loss scale: 1.0 | grad norm: 30.650 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 13.53 +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 13.74 +Number of parameters in most loaded shard in billions: 13.7356 +Theoretical memory footprints: weight and optimizer=98244.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 66144.1328125 | max allocated: 124949.08837890625 | reserved: 127502.0 | max reserved: 127502.0 + [2026-08-26 19:30:44] iteration 2/ 6 | consumed samples: 16 | elapsed time per iteration (ms): 849.6 | throughput per GPU (TFLOP/s/GPU): 150.2 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.091986E+01 | loss scale: 1.0 | grad norm: 30.565 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:30:45] iteration 3/ 6 | consumed samples: 24 | elapsed time per iteration (ms): 822.2 | throughput per GPU (TFLOP/s/GPU): 155.2 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.956084E+00 | loss scale: 1.0 | grad norm: 42.533 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 19:30:47] iteration 4/ 6 | consumed samples: 32 | elapsed time per iteration (ms): 2127.1 | throughput per GPU (TFLOP/s/GPU): 60.0 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.478545E+00 | loss scale: 1.0 | grad norm: 7.152 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:30:48] iteration 5/ 6 | consumed samples: 40 | elapsed time per iteration (ms): 814.7 | throughput per GPU (TFLOP/s/GPU): 156.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.091896E+01 | loss scale: 1.0 | grad norm: 452.732 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:30:49] iteration 6/ 6 | consumed samples: 48 | elapsed time per iteration (ms): 767.9 | throughput per GPU (TFLOP/s/GPU): 166.1 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.375167E+00 | loss scale: 1.0 | grad norm: 4.992 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 19:30:49 +[rank0]:[W826 19:30:50.027560372 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 19:30:52.308000 679520 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 19:30:52.309000 679520 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 679643 closing signal SIGTERM +W0826 19:30:52.309000 679520 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 679644 closing signal SIGTERM +W0826 19:30:52.312000 679520 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 679645 closing signal SIGTERM +W0826 19:30:52.319000 679520 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 679646 closing signal SIGTERM +W0826 19:30:52.321000 679520 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 679647 closing signal SIGTERM +W0826 19:30:52.323000 679520 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 679648 closing signal SIGTERM +W0826 19:30:52.324000 679520 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 679649 closing signal SIGTERM +W0826 19:30:52.328000 679520 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 679650 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 679520 got signal: 15 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/mem_samples.csv b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..b4332b2d5db6af01fd98c23be1e1af0fc3b0ecd8 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/mem_samples.csv @@ -0,0 +1,208 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 819 +2, 819 +3, 819 +4, 819 +5, 819 +6, 819 +7, 819 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 2471 +1, 2471 +2, 2471 +3, 2471 +4, 2471 +5, 2471 +6, 2471 +7, 2471 +0, 6607 +1, 6519 +2, 6607 +3, 6607 +4, 6607 +5, 6607 +6, 6607 +7, 6607 +0, 35381 +1, 35505 +2, 35375 +3, 35499 +4, 35451 +5, 35435 +6, 35423 +7, 35505 +0, 41333 +1, 41455 +2, 41329 +3, 41387 +4, 41339 +5, 41385 +6, 41375 +7, 41455 +0, 46289 +1, 46409 +2, 46283 +3, 46341 +4, 46295 +5, 46339 +6, 46331 +7, 46409 +0, 46289 +1, 46409 +2, 46283 +3, 46341 +4, 46295 +5, 46339 +6, 46331 +7, 46409 +0, 46321 +1, 46441 +2, 46315 +3, 46373 +4, 46327 +5, 46371 +6, 46363 +7, 46441 +0, 116085 +1, 116077 +2, 116079 +3, 116075 +4, 116089 +5, 116071 +6, 116063 +7, 116077 +0, 139411 +1, 139405 +2, 139405 +3, 139401 +4, 139417 +5, 139399 +6, 140175 +7, 139405 +0, 139801 +1, 139793 +2, 139795 +3, 139791 +4, 139805 +5, 139787 +6, 140565 +7, 139793 +0, 141851 +1, 141843 +2, 141845 +3, 141841 +4, 141855 +5, 141837 +6, 142615 +7, 141843 +0, 142499 +1, 142491 +2, 142493 +3, 142489 +4, 142503 +5, 142485 +6, 143263 +7, 142491 +0, 143105 +1, 143147 +2, 143113 +3, 143087 +4, 143143 +5, 143091 +6, 143873 +7, 143119 +0, 157147 +1, 157135 +2, 157143 +3, 157135 +4, 157153 +5, 157129 +6, 156339 +7, 157135 +0, 157147 +1, 157135 +2, 157143 +3, 157135 +4, 157153 +5, 157129 +6, 156339 +7, 157135 +0, 155911 +1, 156287 +2, 155907 +3, 155899 +4, 155917 +5, 155893 +6, 155103 +7, 156287 +0, 10528 +1, 154995 +2, 154871 +3, 154475 +4, 154881 +5, 154857 +6, 154067 +7, 154863 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T193150674812Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T193150674812Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3d980a0ae30a68d27abe8285c3cb5d69b74752db --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T193150674812Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69289811456, + "active_bytes_current_delta": 45492018688, + "active_bytes_peak": 131018615296, + "active_bytes_peak_delta": 107208239616, + "after": { + "active_bytes_current": 69289811456, + "active_bytes_peak": 131018615296, + "allocated_bytes_current": 69289811456, + "allocated_bytes_peak": 131018615296, + "device": 0, + "inactive_split_bytes_current": 9003164160, + "inactive_split_bytes_peak": 22463227904, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 133695537152, + "reserved_bytes_peak": 133695537152, + "segment_count_current": 1121, + "segment_count_peak": 1121, + "timestamp_utc": "2026-08-26T19:31:50.674246Z", + "world_size": 8 + }, + "allocated_bytes_current": 69289811456, + "allocated_bytes_current_delta": 45492018688, + "allocated_bytes_peak": 131018615296, + "allocated_bytes_peak_delta": 107208239616, + "before": { + "active_bytes_current": 23797792768, + "active_bytes_peak": 23810375680, + "allocated_bytes_current": 23797792768, + "allocated_bytes_peak": 23810375680, + "device": 0, + "inactive_split_bytes_current": 768245760, + "inactive_split_bytes_peak": 1283983872, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 24582815744, + "reserved_bytes_peak": 24582815744, + "segment_count_current": 470, + "segment_count_peak": 470, + "timestamp_utc": "2026-08-26T19:31:27.995245Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9003164160, + "inactive_split_bytes_current_delta": 8234918400, + "inactive_split_bytes_peak": 22463227904, + "inactive_split_bytes_peak_delta": 21179244032, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 133695537152, + "reserved_bytes_current_delta": 109112721408, + "reserved_bytes_peak": 133695537152, + "reserved_bytes_peak_delta": 109112721408, + "segment_count_current": 1121, + "segment_count_current_delta": 651, + "segment_count_peak": 1121, + "segment_count_peak_delta": 651, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260826T193150674812Z.pickle", + "timestamp_utc": "2026-08-26T19:31:50.783259Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T193150674812Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T193150674812Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..98a35d9daedb4858db16bd4e91245c8a6a3d4e42 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260826T193150674812Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd9fb707a6bba889c02db5fa9313cadeb1a70f773063de9f718e23105c56e5cc +size 4775203 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T193151565874Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T193151565874Z.json new file mode 100644 index 0000000000000000000000000000000000000000..59d84f0a6b8fe77481f38eec4198a404d768f392 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T193151565874Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69289811456, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145252282880, + "active_bytes_peak_delta": 14233667584, + "after": { + "active_bytes_current": 69289811456, + "active_bytes_peak": 145252282880, + "allocated_bytes_current": 69289811456, + "allocated_bytes_peak": 145252282880, + "device": 0, + "inactive_split_bytes_current": 9003164160, + "inactive_split_bytes_peak": 22463227904, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 148239286272, + "reserved_bytes_peak": 148239286272, + "segment_count_current": 1133, + "segment_count_peak": 1133, + "timestamp_utc": "2026-08-26T19:31:51.565450Z", + "world_size": 8 + }, + "allocated_bytes_current": 69289811456, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145252282880, + "allocated_bytes_peak_delta": 14233667584, + "before": { + "active_bytes_current": 69289811456, + "active_bytes_peak": 131018615296, + "allocated_bytes_current": 69289811456, + "allocated_bytes_peak": 131018615296, + "device": 0, + "inactive_split_bytes_current": 9003164160, + "inactive_split_bytes_peak": 22463227904, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 133695537152, + "reserved_bytes_peak": 133695537152, + "segment_count_current": 1121, + "segment_count_peak": 1121, + "timestamp_utc": "2026-08-26T19:31:50.791644Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9003164160, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22463227904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 148239286272, + "reserved_bytes_current_delta": 14543749120, + "reserved_bytes_peak": 148239286272, + "reserved_bytes_peak_delta": 14543749120, + "segment_count_current": 1133, + "segment_count_current_delta": 12, + "segment_count_peak": 1133, + "segment_count_peak_delta": 12, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260826T193151565874Z.pickle", + "timestamp_utc": "2026-08-26T19:31:51.657264Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T193151565874Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T193151565874Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ed5a69e5f836ec1859be037e79f4122ad995170a --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260826T193151565874Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54205cb976706043f6345651325d80f08948af12da96334098d60295c108d1eb +size 4038659 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T193152432027Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T193152432027Z.json new file mode 100644 index 0000000000000000000000000000000000000000..55444b1871d70c373d7956f756ce0bf382773640 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T193152432027Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69289811456, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145252282880, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69289811456, + "active_bytes_peak": 145252282880, + "allocated_bytes_current": 69289811456, + "allocated_bytes_peak": 145252282880, + "device": 0, + "inactive_split_bytes_current": 9003164160, + "inactive_split_bytes_peak": 22463227904, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 148239286272, + "reserved_bytes_peak": 148239286272, + "segment_count_current": 1133, + "segment_count_peak": 1133, + "timestamp_utc": "2026-08-26T19:31:52.431627Z", + "world_size": 8 + }, + "allocated_bytes_current": 69289811456, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145252282880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69289811456, + "active_bytes_peak": 145252282880, + "allocated_bytes_current": 69289811456, + "allocated_bytes_peak": 145252282880, + "device": 0, + "inactive_split_bytes_current": 9003164160, + "inactive_split_bytes_peak": 22463227904, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 148239286272, + "reserved_bytes_peak": 148239286272, + "segment_count_current": 1133, + "segment_count_peak": 1133, + "timestamp_utc": "2026-08-26T19:31:51.664148Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9003164160, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22463227904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 148239286272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148239286272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1133, + "segment_count_current_delta": 0, + "segment_count_peak": 1133, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260826T193152432027Z.pickle", + "timestamp_utc": "2026-08-26T19:31:52.515131Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T193152432027Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T193152432027Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a488923fb477aff35e9874192404d35a628140d9 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260826T193152432027Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd0bdd3b0d7b84f8124421e11fc6278493f8ca66db8c224cd2dfd08ca69fda74 +size 4038900 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T193153293151Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T193153293151Z.json new file mode 100644 index 0000000000000000000000000000000000000000..953b06681a232d5cff1b063a624595f14d6473b7 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T193153293151Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69289811456, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145252282880, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69289811456, + "active_bytes_peak": 145252282880, + "allocated_bytes_current": 69289811456, + "allocated_bytes_peak": 145252282880, + "device": 0, + "inactive_split_bytes_current": 9003164160, + "inactive_split_bytes_peak": 22463227904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 148239286272, + "reserved_bytes_peak": 148239286272, + "segment_count_current": 1133, + "segment_count_peak": 1133, + "timestamp_utc": "2026-08-26T19:31:53.292758Z", + "world_size": 8 + }, + "allocated_bytes_current": 69289811456, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145252282880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69289811456, + "active_bytes_peak": 145252282880, + "allocated_bytes_current": 69289811456, + "allocated_bytes_peak": 145252282880, + "device": 0, + "inactive_split_bytes_current": 9003164160, + "inactive_split_bytes_peak": 22463227904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 148239286272, + "reserved_bytes_peak": 148239286272, + "segment_count_current": 1133, + "segment_count_peak": 1133, + "timestamp_utc": "2026-08-26T19:31:52.521659Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9003164160, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22463227904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 148239286272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148239286272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1133, + "segment_count_current_delta": 0, + "segment_count_peak": 1133, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260826T193153293151Z.pickle", + "timestamp_utc": "2026-08-26T19:31:53.375899Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T193153293151Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T193153293151Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2a4710029205fbf61d8345d45a1b119d61f62326 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260826T193153293151Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c79a82315f4defd37d8d4551c84139d54039598cdcbc4ddc87926c8270e5bc8 +size 4038832 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T193154154155Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T193154154155Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9d5ee246dbd658dd943d1495ebaa483d8c6df782 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T193154154155Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69289811456, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145252282880, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69289811456, + "active_bytes_peak": 145252282880, + "allocated_bytes_current": 69289811456, + "allocated_bytes_peak": 145252282880, + "device": 0, + "inactive_split_bytes_current": 9003164160, + "inactive_split_bytes_peak": 22463227904, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 148239286272, + "reserved_bytes_peak": 148239286272, + "segment_count_current": 1133, + "segment_count_peak": 1133, + "timestamp_utc": "2026-08-26T19:31:54.153765Z", + "world_size": 8 + }, + "allocated_bytes_current": 69289811456, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145252282880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69289811456, + "active_bytes_peak": 145252282880, + "allocated_bytes_current": 69289811456, + "allocated_bytes_peak": 145252282880, + "device": 0, + "inactive_split_bytes_current": 9003164160, + "inactive_split_bytes_peak": 22463227904, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 148239286272, + "reserved_bytes_peak": 148239286272, + "segment_count_current": 1133, + "segment_count_peak": 1133, + "timestamp_utc": "2026-08-26T19:31:53.382323Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9003164160, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22463227904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 148239286272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148239286272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1133, + "segment_count_current_delta": 0, + "segment_count_peak": 1133, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260826T193154154155Z.pickle", + "timestamp_utc": "2026-08-26T19:31:54.236208Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T193154154155Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T193154154155Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a578e864aa2a4394d5b9f4ad5b5f3a099685113a --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260826T193154154155Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39c11d99209994776d422a1895176cd3dc15d52e7c002b5a5748e77be540a414 +size 4038931 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T193150674662Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T193150674662Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cede0e9d867091963b65bbc8ec1b432331d5bb8a --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T193150674662Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69299097600, + "active_bytes_current_delta": 45496446976, + "active_bytes_peak": 131036168704, + "active_bytes_peak_delta": 107220935168, + "after": { + "active_bytes_current": 69299097600, + "active_bytes_peak": 131036168704, + "allocated_bytes_current": 69299097600, + "allocated_bytes_peak": 131036168704, + "device": 1, + "inactive_split_bytes_current": 8989683712, + "inactive_split_bytes_peak": 22446469120, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 133687148544, + "reserved_bytes_peak": 133687148544, + "segment_count_current": 1127, + "segment_count_peak": 1127, + "timestamp_utc": "2026-08-26T19:31:50.674181Z", + "world_size": 8 + }, + "allocated_bytes_current": 69299097600, + "allocated_bytes_current_delta": 45496446976, + "allocated_bytes_peak": 131036168704, + "allocated_bytes_peak_delta": 107220935168, + "before": { + "active_bytes_current": 23802650624, + "active_bytes_peak": 23815233536, + "allocated_bytes_current": 23802650624, + "allocated_bytes_peak": 23815233536, + "device": 1, + "inactive_split_bytes_current": 891314176, + "inactive_split_bytes_peak": 1283983872, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 24712839168, + "reserved_bytes_peak": 24712839168, + "segment_count_current": 477, + "segment_count_peak": 477, + "timestamp_utc": "2026-08-26T19:31:27.995030Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8989683712, + "inactive_split_bytes_current_delta": 8098369536, + "inactive_split_bytes_peak": 22446469120, + "inactive_split_bytes_peak_delta": 21162485248, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 133687148544, + "reserved_bytes_current_delta": 108974309376, + "reserved_bytes_peak": 133687148544, + "reserved_bytes_peak_delta": 108974309376, + "segment_count_current": 1127, + "segment_count_current_delta": 650, + "segment_count_peak": 1127, + "segment_count_peak_delta": 650, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260826T193150674662Z.pickle", + "timestamp_utc": "2026-08-26T19:31:50.781785Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T193150674662Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T193150674662Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..26edec0fda014b4995a530cf54aee0c5cf2ac705 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000000_20260826T193150674662Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99cdd5562cfae58c32d0e4a194ca808384b19fe9cacec70f4ac85daec1a1a0d7 +size 4746653 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T193151565838Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T193151565838Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7c8b6d75b505783bc0c35245d6365368a018cc7e --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T193151565838Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69299097600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145265927168, + "active_bytes_peak_delta": 14229758464, + "after": { + "active_bytes_current": 69299097600, + "active_bytes_peak": 145265927168, + "allocated_bytes_current": 69299097600, + "allocated_bytes_peak": 145265927168, + "device": 1, + "inactive_split_bytes_current": 8989683712, + "inactive_split_bytes_peak": 22446469120, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:31:51.565433Z", + "world_size": 8 + }, + "allocated_bytes_current": 69299097600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145265927168, + "allocated_bytes_peak_delta": 14229758464, + "before": { + "active_bytes_current": 69299097600, + "active_bytes_peak": 131036168704, + "allocated_bytes_current": 69299097600, + "allocated_bytes_peak": 131036168704, + "device": 1, + "inactive_split_bytes_current": 8989683712, + "inactive_split_bytes_peak": 22446469120, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 133687148544, + "reserved_bytes_peak": 133687148544, + "segment_count_current": 1127, + "segment_count_peak": 1127, + "timestamp_utc": "2026-08-26T19:31:50.790091Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8989683712, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22446469120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 148226703360, + "reserved_bytes_current_delta": 14539554816, + "reserved_bytes_peak": 148226703360, + "reserved_bytes_peak_delta": 14539554816, + "segment_count_current": 1137, + "segment_count_current_delta": 10, + "segment_count_peak": 1137, + "segment_count_peak_delta": 10, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260826T193151565838Z.pickle", + "timestamp_utc": "2026-08-26T19:31:51.654538Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T193151565838Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T193151565838Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fd4d0ef6be498adedd20700d163e2d1b9bb05e20 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000001_20260826T193151565838Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c19060fbf4dd762ebfe35f5bd3374342c1cb6736d3dcf5cf6f34a1e6bcd2e94 +size 4011977 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T193152431988Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T193152431988Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fc26944d2047e42db9ea1560ed61474e3ae64126 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T193152431988Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69299097600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145265927168, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69299097600, + "active_bytes_peak": 145265927168, + "allocated_bytes_current": 69299097600, + "allocated_bytes_peak": 145265927168, + "device": 1, + "inactive_split_bytes_current": 8989683712, + "inactive_split_bytes_peak": 22446469120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:31:52.431593Z", + "world_size": 8 + }, + "allocated_bytes_current": 69299097600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145265927168, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69299097600, + "active_bytes_peak": 145265927168, + "allocated_bytes_current": 69299097600, + "allocated_bytes_peak": 145265927168, + "device": 1, + "inactive_split_bytes_current": 8989683712, + "inactive_split_bytes_peak": 22446469120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:31:51.661071Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8989683712, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22446469120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 148226703360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148226703360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1137, + "segment_count_current_delta": 0, + "segment_count_peak": 1137, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260826T193152431988Z.pickle", + "timestamp_utc": "2026-08-26T19:31:52.511122Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T193152431988Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T193152431988Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b3a6bc0fc574f5814ca64aa585039c09b38c2099 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T193152431988Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd1b31c296a8bbe98ad8887bd8fcabef77757e769d1ceef0850033e022eeac5a +size 4012271 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T193153293062Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T193153293062Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4b4d6cb4e02070f92b4276a7522fcdf37e1c2e4b --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T193153293062Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69299097600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145265927168, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69299097600, + "active_bytes_peak": 145265927168, + "allocated_bytes_current": 69299097600, + "allocated_bytes_peak": 145265927168, + "device": 1, + "inactive_split_bytes_current": 8989683712, + "inactive_split_bytes_peak": 22446469120, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:31:53.292672Z", + "world_size": 8 + }, + "allocated_bytes_current": 69299097600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145265927168, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69299097600, + "active_bytes_peak": 145265927168, + "allocated_bytes_current": 69299097600, + "allocated_bytes_peak": 145265927168, + "device": 1, + "inactive_split_bytes_current": 8989683712, + "inactive_split_bytes_peak": 22446469120, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:31:52.517197Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8989683712, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22446469120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 148226703360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148226703360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1137, + "segment_count_current_delta": 0, + "segment_count_peak": 1137, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T193153293062Z.pickle", + "timestamp_utc": "2026-08-26T19:31:53.374263Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T193153293062Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T193153293062Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf4587b057144d8132f7744062ff1475403307cf --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T193153293062Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b502b643614a41b1c22136a7dbc675a5caa28233c2af09aaed57d5b7fb026f7 +size 4012269 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T193154154011Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T193154154011Z.json new file mode 100644 index 0000000000000000000000000000000000000000..779d11355a540abb8b88613ff7361fadcb13ba3c --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T193154154011Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69299097600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145265927168, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69299097600, + "active_bytes_peak": 145265927168, + "allocated_bytes_current": 69299097600, + "allocated_bytes_peak": 145265927168, + "device": 1, + "inactive_split_bytes_current": 8989683712, + "inactive_split_bytes_peak": 22446469120, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:31:54.153624Z", + "world_size": 8 + }, + "allocated_bytes_current": 69299097600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145265927168, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69299097600, + "active_bytes_peak": 145265927168, + "allocated_bytes_current": 69299097600, + "allocated_bytes_peak": 145265927168, + "device": 1, + "inactive_split_bytes_current": 8989683712, + "inactive_split_bytes_peak": 22446469120, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:31:53.380497Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8989683712, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22446469120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 148226703360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148226703360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1137, + "segment_count_current_delta": 0, + "segment_count_peak": 1137, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T193154154011Z.pickle", + "timestamp_utc": "2026-08-26T19:31:54.233315Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T193154154011Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T193154154011Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..53d6a68cd55974102beda5f65543df3b64b9fa8c --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T193154154011Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ded02c8e670e7ba136325080e016f1e3394a1dfbcb63eb30f9767f76c794e880 +size 4012311 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T193150674651Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T193150674651Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ecc9acb38430666963371aab635f014048fbf4c0 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T193150674651Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69297611264, + "active_bytes_current_delta": 45489928704, + "active_bytes_peak": 131032331776, + "active_bytes_peak_delta": 107212066304, + "after": { + "active_bytes_current": 69297611264, + "active_bytes_peak": 131032331776, + "allocated_bytes_current": 69297611264, + "allocated_bytes_peak": 131032331776, + "device": 2, + "inactive_split_bytes_current": 8993267200, + "inactive_split_bytes_peak": 22522369024, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 133689245696, + "reserved_bytes_peak": 133689245696, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:31:50.674158Z", + "world_size": 8 + }, + "allocated_bytes_current": 69297611264, + "allocated_bytes_current_delta": 45489928704, + "allocated_bytes_peak": 131032331776, + "allocated_bytes_peak_delta": 107212066304, + "before": { + "active_bytes_current": 23807682560, + "active_bytes_peak": 23820265472, + "allocated_bytes_current": 23807682560, + "allocated_bytes_peak": 23820265472, + "device": 2, + "inactive_split_bytes_current": 754161664, + "inactive_split_bytes_peak": 1283983872, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 24576524288, + "reserved_bytes_peak": 24576524288, + "segment_count_current": 472, + "segment_count_peak": 472, + "timestamp_utc": "2026-08-26T19:31:27.995178Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8993267200, + "inactive_split_bytes_current_delta": 8239105536, + "inactive_split_bytes_peak": 22522369024, + "inactive_split_bytes_peak_delta": 21238385152, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 133689245696, + "reserved_bytes_current_delta": 109112721408, + "reserved_bytes_peak": 133689245696, + "reserved_bytes_peak_delta": 109112721408, + "segment_count_current": 1123, + "segment_count_current_delta": 651, + "segment_count_peak": 1123, + "segment_count_peak_delta": 651, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T193150674651Z.pickle", + "timestamp_utc": "2026-08-26T19:31:50.785374Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T193150674651Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T193150674651Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..28fb41b9537f6c2be325ca84d2b12ef0729f9a72 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T193150674651Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:923517d38552bdaa4e9d8fd990adbc98c3caf89e11b0637afc0cc13e6372b0df +size 4758022 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T193151565907Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T193151565907Z.json new file mode 100644 index 0000000000000000000000000000000000000000..936ac44c48eb97fe5fc7fd889695784c152604ba --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T193151565907Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69297611264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145251845632, + "active_bytes_peak_delta": 14219513856, + "after": { + "active_bytes_current": 69297611264, + "active_bytes_peak": 145251845632, + "allocated_bytes_current": 69297611264, + "allocated_bytes_peak": 145251845632, + "device": 2, + "inactive_split_bytes_current": 8993267200, + "inactive_split_bytes_peak": 22522369024, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 148235091968, + "reserved_bytes_peak": 148235091968, + "segment_count_current": 1136, + "segment_count_peak": 1136, + "timestamp_utc": "2026-08-26T19:31:51.565471Z", + "world_size": 8 + }, + "allocated_bytes_current": 69297611264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145251845632, + "allocated_bytes_peak_delta": 14219513856, + "before": { + "active_bytes_current": 69297611264, + "active_bytes_peak": 131032331776, + "allocated_bytes_current": 69297611264, + "allocated_bytes_peak": 131032331776, + "device": 2, + "inactive_split_bytes_current": 8993267200, + "inactive_split_bytes_peak": 22522369024, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 133689245696, + "reserved_bytes_peak": 133689245696, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:31:50.794437Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8993267200, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22522369024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 148235091968, + "reserved_bytes_current_delta": 14545846272, + "reserved_bytes_peak": 148235091968, + "reserved_bytes_peak_delta": 14545846272, + "segment_count_current": 1136, + "segment_count_current_delta": 13, + "segment_count_peak": 1136, + "segment_count_peak_delta": 13, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T193151565907Z.pickle", + "timestamp_utc": "2026-08-26T19:31:51.660789Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T193151565907Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T193151565907Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1ef066d0b8d8add49be6f8366bec134cc4e06d29 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T193151565907Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35091cf0b7faf1b5e245bb327167a70fa783925f44ad962acd5b47875b5bdf8c +size 4019737 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T193152432043Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T193152432043Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8b376d0c7bbf372acd6f03ecfd982c7ae213fc69 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T193152432043Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69297611264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145252894208, + "active_bytes_peak_delta": 1048576, + "after": { + "active_bytes_current": 69297611264, + "active_bytes_peak": 145252894208, + "allocated_bytes_current": 69297611264, + "allocated_bytes_peak": 145252894208, + "device": 2, + "inactive_split_bytes_current": 8993267200, + "inactive_split_bytes_peak": 22522369024, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 148235091968, + "reserved_bytes_peak": 148235091968, + "segment_count_current": 1136, + "segment_count_peak": 1136, + "timestamp_utc": "2026-08-26T19:31:52.431639Z", + "world_size": 8 + }, + "allocated_bytes_current": 69297611264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145252894208, + "allocated_bytes_peak_delta": 1048576, + "before": { + "active_bytes_current": 69297611264, + "active_bytes_peak": 145251845632, + "allocated_bytes_current": 69297611264, + "allocated_bytes_peak": 145251845632, + "device": 2, + "inactive_split_bytes_current": 8993267200, + "inactive_split_bytes_peak": 22522369024, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 148235091968, + "reserved_bytes_peak": 148235091968, + "segment_count_current": 1136, + "segment_count_peak": 1136, + "timestamp_utc": "2026-08-26T19:31:51.668085Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8993267200, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22522369024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 148235091968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148235091968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1136, + "segment_count_current_delta": 0, + "segment_count_peak": 1136, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T193152432043Z.pickle", + "timestamp_utc": "2026-08-26T19:31:52.518566Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T193152432043Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T193152432043Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..95c8b42903358e8e0864f53e5bfd45e99820c523 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T193152432043Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdc19f0fb48498562c5604c4dd9169b4d7d63b6b1e702254e838f5c1f23e386c +size 4019874 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T193153293134Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T193153293134Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0fcf7da010f317571fa54963b4873a4b8341a6da --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T193153293134Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69297611264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145252894208, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69297611264, + "active_bytes_peak": 145252894208, + "allocated_bytes_current": 69297611264, + "allocated_bytes_peak": 145252894208, + "device": 2, + "inactive_split_bytes_current": 8993267200, + "inactive_split_bytes_peak": 22522369024, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 148235091968, + "reserved_bytes_peak": 148235091968, + "segment_count_current": 1136, + "segment_count_peak": 1136, + "timestamp_utc": "2026-08-26T19:31:53.292728Z", + "world_size": 8 + }, + "allocated_bytes_current": 69297611264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145252894208, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69297611264, + "active_bytes_peak": 145252894208, + "allocated_bytes_current": 69297611264, + "allocated_bytes_peak": 145252894208, + "device": 2, + "inactive_split_bytes_current": 8993267200, + "inactive_split_bytes_peak": 22522369024, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 148235091968, + "reserved_bytes_peak": 148235091968, + "segment_count_current": 1136, + "segment_count_peak": 1136, + "timestamp_utc": "2026-08-26T19:31:52.525341Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8993267200, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22522369024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 148235091968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148235091968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1136, + "segment_count_current_delta": 0, + "segment_count_peak": 1136, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T193153293134Z.pickle", + "timestamp_utc": "2026-08-26T19:31:53.380072Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T193153293134Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T193153293134Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7e2217af85732770d8ece6cad6e68fd8d29565cc --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T193153293134Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a37dd1ef9d2872968ff3346d039bd1602bb88dc38790999755dab71748183e43 +size 4019806 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T193154154075Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T193154154075Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9506bbeeed4d427497fe8e6b9265ec2d367df563 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T193154154075Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69297611264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145252894208, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69297611264, + "active_bytes_peak": 145252894208, + "allocated_bytes_current": 69297611264, + "allocated_bytes_peak": 145252894208, + "device": 2, + "inactive_split_bytes_current": 8993267200, + "inactive_split_bytes_peak": 22522369024, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 148235091968, + "reserved_bytes_peak": 148235091968, + "segment_count_current": 1136, + "segment_count_peak": 1136, + "timestamp_utc": "2026-08-26T19:31:54.153681Z", + "world_size": 8 + }, + "allocated_bytes_current": 69297611264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145252894208, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69297611264, + "active_bytes_peak": 145252894208, + "allocated_bytes_current": 69297611264, + "allocated_bytes_peak": 145252894208, + "device": 2, + "inactive_split_bytes_current": 8993267200, + "inactive_split_bytes_peak": 22522369024, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 148235091968, + "reserved_bytes_peak": 148235091968, + "segment_count_current": 1136, + "segment_count_peak": 1136, + "timestamp_utc": "2026-08-26T19:31:53.386878Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8993267200, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22522369024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 148235091968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148235091968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1136, + "segment_count_current_delta": 0, + "segment_count_peak": 1136, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T193154154075Z.pickle", + "timestamp_utc": "2026-08-26T19:31:54.241160Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T193154154075Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T193154154075Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a2cd90c0ec0dbbf1823f5bd52aea14cd68c79a0b --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T193154154075Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1e5bace85debc968fef6c7b21ce30b899d6d255dbf4976bca9130b15e3c6281 +size 4019874 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T193150674644Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T193150674644Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d723f8cdc85504015ec1cc970028410df96083d8 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T193150674644Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69291905024, + "active_bytes_current_delta": 45480647168, + "active_bytes_peak": 131031941632, + "active_bytes_peak_delta": 107208100864, + "after": { + "active_bytes_current": 69291905024, + "active_bytes_peak": 131031941632, + "allocated_bytes_current": 69291905024, + "allocated_bytes_peak": 131031941632, + "device": 3, + "inactive_split_bytes_current": 8990584832, + "inactive_split_bytes_peak": 22507393024, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 133685051392, + "reserved_bytes_peak": 133685051392, + "segment_count_current": 1127, + "segment_count_peak": 1127, + "timestamp_utc": "2026-08-26T19:31:50.674114Z", + "world_size": 8 + }, + "allocated_bytes_current": 69291905024, + "allocated_bytes_current_delta": 45480647168, + "allocated_bytes_peak": 131031941632, + "allocated_bytes_peak_delta": 107208100864, + "before": { + "active_bytes_current": 23811257856, + "active_bytes_peak": 23823840768, + "allocated_bytes_current": 23811257856, + "allocated_bytes_peak": 23823840768, + "device": 3, + "inactive_split_bytes_current": 811403776, + "inactive_split_bytes_peak": 1283983872, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 24639438848, + "reserved_bytes_peak": 24639438848, + "segment_count_current": 477, + "segment_count_peak": 477, + "timestamp_utc": "2026-08-26T19:31:27.995054Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8990584832, + "inactive_split_bytes_current_delta": 8179181056, + "inactive_split_bytes_peak": 22507393024, + "inactive_split_bytes_peak_delta": 21223409152, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 133685051392, + "reserved_bytes_current_delta": 109045612544, + "reserved_bytes_peak": 133685051392, + "reserved_bytes_peak_delta": 109045612544, + "segment_count_current": 1127, + "segment_count_current_delta": 650, + "segment_count_peak": 1127, + "segment_count_peak_delta": 650, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T193150674644Z.pickle", + "timestamp_utc": "2026-08-26T19:31:50.787697Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T193150674644Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T193150674644Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d2881ff519beb1ebbd8d4ecd5e30da4f3f061275 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T193150674644Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:968dbfb9155c1d6cf26d47fca6cd578e737d21ab538dc652e8e4a073e2578ec3 +size 5166962 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T193151565883Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T193151565883Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7f70481684109ff026d847805be58b74c2327361 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T193151565883Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69291905024, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145255880192, + "active_bytes_peak_delta": 14223938560, + "after": { + "active_bytes_current": 69291905024, + "active_bytes_peak": 145255880192, + "allocated_bytes_current": 69291905024, + "allocated_bytes_peak": 145255880192, + "device": 3, + "inactive_split_bytes_current": 8990584832, + "inactive_split_bytes_peak": 22507393024, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1138, + "segment_count_peak": 1138, + "timestamp_utc": "2026-08-26T19:31:51.565479Z", + "world_size": 8 + }, + "allocated_bytes_current": 69291905024, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145255880192, + "allocated_bytes_peak_delta": 14223938560, + "before": { + "active_bytes_current": 69291905024, + "active_bytes_peak": 131031941632, + "allocated_bytes_current": 69291905024, + "allocated_bytes_peak": 131031941632, + "device": 3, + "inactive_split_bytes_current": 8990584832, + "inactive_split_bytes_peak": 22507393024, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 133685051392, + "reserved_bytes_peak": 133685051392, + "segment_count_current": 1127, + "segment_count_peak": 1127, + "timestamp_utc": "2026-08-26T19:31:50.796651Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8990584832, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22507393024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 148226703360, + "reserved_bytes_current_delta": 14541651968, + "reserved_bytes_peak": 148226703360, + "reserved_bytes_peak_delta": 14541651968, + "segment_count_current": 1138, + "segment_count_current_delta": 11, + "segment_count_peak": 1138, + "segment_count_peak_delta": 11, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T193151565883Z.pickle", + "timestamp_utc": "2026-08-26T19:31:51.656853Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T193151565883Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T193151565883Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f57371456f4e72a1edd8e69f6393f1d8705a9586 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T193151565883Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a47977d2ad560a91286d90ad20d9f16233e3f4376db012dae45e4edb87b9ae5 +size 4431911 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T193152432094Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T193152432094Z.json new file mode 100644 index 0000000000000000000000000000000000000000..305280d790d53845669d647944407926ced0a932 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T193152432094Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69291905024, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145258100736, + "active_bytes_peak_delta": 2220544, + "after": { + "active_bytes_current": 69291905024, + "active_bytes_peak": 145258100736, + "allocated_bytes_current": 69291905024, + "allocated_bytes_peak": 145258100736, + "device": 3, + "inactive_split_bytes_current": 8990584832, + "inactive_split_bytes_peak": 22507393024, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1138, + "segment_count_peak": 1138, + "timestamp_utc": "2026-08-26T19:31:52.431697Z", + "world_size": 8 + }, + "allocated_bytes_current": 69291905024, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145258100736, + "allocated_bytes_peak_delta": 2220544, + "before": { + "active_bytes_current": 69291905024, + "active_bytes_peak": 145255880192, + "allocated_bytes_current": 69291905024, + "allocated_bytes_peak": 145255880192, + "device": 3, + "inactive_split_bytes_current": 8990584832, + "inactive_split_bytes_peak": 22507393024, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1138, + "segment_count_peak": 1138, + "timestamp_utc": "2026-08-26T19:31:51.663879Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8990584832, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22507393024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 148226703360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148226703360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1138, + "segment_count_current_delta": 0, + "segment_count_peak": 1138, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T193152432094Z.pickle", + "timestamp_utc": "2026-08-26T19:31:52.515847Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T193152432094Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T193152432094Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..86502f0c000aeaa7cb1754a1841a225e0e8ba98c --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T193152432094Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dd72920987e37efda01a13152c0bf816631baac1db332e5a3a856bd529da828 +size 4431975 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T193153293166Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T193153293166Z.json new file mode 100644 index 0000000000000000000000000000000000000000..64140a68c33ebba28d372e36145ca1b208db0723 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T193153293166Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69291905024, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145258100736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69291905024, + "active_bytes_peak": 145258100736, + "allocated_bytes_current": 69291905024, + "allocated_bytes_peak": 145258100736, + "device": 3, + "inactive_split_bytes_current": 8990584832, + "inactive_split_bytes_peak": 22507393024, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1138, + "segment_count_peak": 1138, + "timestamp_utc": "2026-08-26T19:31:53.292773Z", + "world_size": 8 + }, + "allocated_bytes_current": 69291905024, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145258100736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69291905024, + "active_bytes_peak": 145258100736, + "allocated_bytes_current": 69291905024, + "allocated_bytes_peak": 145258100736, + "device": 3, + "inactive_split_bytes_current": 8990584832, + "inactive_split_bytes_peak": 22507393024, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1138, + "segment_count_peak": 1138, + "timestamp_utc": "2026-08-26T19:31:52.522349Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8990584832, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22507393024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 148226703360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148226703360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1138, + "segment_count_current_delta": 0, + "segment_count_peak": 1138, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T193153293166Z.pickle", + "timestamp_utc": "2026-08-26T19:31:53.376941Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T193153293166Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T193153293166Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e18ee1a76bd7bd64032d0604794bb85f09b0aa1a --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T193153293166Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e928795647250c5248fcf31cb41b7fb82f6d73afb3f7b3d70a1d4c910d020c6d +size 4432128 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T193154154063Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T193154154063Z.json new file mode 100644 index 0000000000000000000000000000000000000000..95c681263bd62e0f71884d9903e483d556ab9703 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T193154154063Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69291905024, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145258100736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69291905024, + "active_bytes_peak": 145258100736, + "allocated_bytes_current": 69291905024, + "allocated_bytes_peak": 145258100736, + "device": 3, + "inactive_split_bytes_current": 8990584832, + "inactive_split_bytes_peak": 22507393024, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1138, + "segment_count_peak": 1138, + "timestamp_utc": "2026-08-26T19:31:54.153672Z", + "world_size": 8 + }, + "allocated_bytes_current": 69291905024, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145258100736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69291905024, + "active_bytes_peak": 145258100736, + "allocated_bytes_current": 69291905024, + "allocated_bytes_peak": 145258100736, + "device": 3, + "inactive_split_bytes_current": 8990584832, + "inactive_split_bytes_peak": 22507393024, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1138, + "segment_count_peak": 1138, + "timestamp_utc": "2026-08-26T19:31:53.383473Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8990584832, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22507393024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 148226703360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148226703360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1138, + "segment_count_current_delta": 0, + "segment_count_peak": 1138, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T193154154063Z.pickle", + "timestamp_utc": "2026-08-26T19:31:54.237007Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T193154154063Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T193154154063Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5afde0e31ecf0d9926fb233923a2dfb6754f2972 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T193154154063Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aefd0028b90386085b276b1aa8d2bcfa51b69f239c6b16791d39b7a941d90821 +size 4432072 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T193150674865Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T193150674865Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a5e5c94541ce19baa772819442ded8dd67639b6e --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T193150674865Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69293369344, + "active_bytes_current_delta": 45489486848, + "active_bytes_peak": 131032443392, + "active_bytes_peak_delta": 107215977984, + "after": { + "active_bytes_current": 69293369344, + "active_bytes_peak": 131032443392, + "allocated_bytes_current": 69293369344, + "allocated_bytes_peak": 131032443392, + "device": 4, + "inactive_split_bytes_current": 9005897728, + "inactive_split_bytes_peak": 22530654720, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 133699731456, + "reserved_bytes_peak": 133699731456, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:31:50.674316Z", + "world_size": 8 + }, + "allocated_bytes_current": 69293369344, + "allocated_bytes_current_delta": 45489486848, + "allocated_bytes_peak": 131032443392, + "allocated_bytes_peak_delta": 107215977984, + "before": { + "active_bytes_current": 23803882496, + "active_bytes_peak": 23816465408, + "allocated_bytes_current": 23803882496, + "allocated_bytes_peak": 23816465408, + "device": 4, + "inactive_split_bytes_current": 768447488, + "inactive_split_bytes_peak": 1283983872, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 24589107200, + "reserved_bytes_peak": 24589107200, + "segment_count_current": 473, + "segment_count_peak": 473, + "timestamp_utc": "2026-08-26T19:31:27.995065Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9005897728, + "inactive_split_bytes_current_delta": 8237450240, + "inactive_split_bytes_peak": 22530654720, + "inactive_split_bytes_peak_delta": 21246670848, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 133699731456, + "reserved_bytes_current_delta": 109110624256, + "reserved_bytes_peak": 133699731456, + "reserved_bytes_peak_delta": 109110624256, + "segment_count_current": 1123, + "segment_count_current_delta": 650, + "segment_count_peak": 1123, + "segment_count_peak_delta": 650, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T193150674865Z.pickle", + "timestamp_utc": "2026-08-26T19:31:50.783092Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T193150674865Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T193150674865Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..80deb5c854ae62c7eaee4825485c84c0713b4d11 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T193150674865Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef737e25d24f76ec3b03784d94d0720ec6ae0b11c8ac59807875d25754bd2acd +size 4756974 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T193151566108Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T193151566108Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bf7d7f600b42566bef3fba1922ca1b421def443d --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T193151566108Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69293369344, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145265893888, + "active_bytes_peak_delta": 14233450496, + "after": { + "active_bytes_current": 69293369344, + "active_bytes_peak": 145265893888, + "allocated_bytes_current": 69293369344, + "allocated_bytes_peak": 145265893888, + "device": 4, + "inactive_split_bytes_current": 9005897728, + "inactive_split_bytes_peak": 22530654720, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 148245577728, + "reserved_bytes_peak": 148245577728, + "segment_count_current": 1136, + "segment_count_peak": 1136, + "timestamp_utc": "2026-08-26T19:31:51.565645Z", + "world_size": 8 + }, + "allocated_bytes_current": 69293369344, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145265893888, + "allocated_bytes_peak_delta": 14233450496, + "before": { + "active_bytes_current": 69293369344, + "active_bytes_peak": 131032443392, + "allocated_bytes_current": 69293369344, + "allocated_bytes_peak": 131032443392, + "device": 4, + "inactive_split_bytes_current": 9005897728, + "inactive_split_bytes_peak": 22530654720, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 133699731456, + "reserved_bytes_peak": 133699731456, + "segment_count_current": 1123, + "segment_count_peak": 1123, + "timestamp_utc": "2026-08-26T19:31:50.791274Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9005897728, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22530654720, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 148245577728, + "reserved_bytes_current_delta": 14545846272, + "reserved_bytes_peak": 148245577728, + "reserved_bytes_peak_delta": 14545846272, + "segment_count_current": 1136, + "segment_count_current_delta": 13, + "segment_count_peak": 1136, + "segment_count_peak_delta": 13, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T193151566108Z.pickle", + "timestamp_utc": "2026-08-26T19:31:51.657008Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T193151566108Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T193151566108Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a95428cdda7ed846481b2d80b3162e888f5a73d3 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T193151566108Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef53fdc09e3ee10d39eb7cce05ce2e6e614398b40a440acb8df705d6b7e554ba +size 4019716 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T193152432189Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T193152432189Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b74dc2f53d5ace7eb0f994155d0eaeaa976a8525 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T193152432189Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69293369344, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145265893888, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69293369344, + "active_bytes_peak": 145265893888, + "allocated_bytes_current": 69293369344, + "allocated_bytes_peak": 145265893888, + "device": 4, + "inactive_split_bytes_current": 9005897728, + "inactive_split_bytes_peak": 22530654720, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 148245577728, + "reserved_bytes_peak": 148245577728, + "segment_count_current": 1136, + "segment_count_peak": 1136, + "timestamp_utc": "2026-08-26T19:31:52.431783Z", + "world_size": 8 + }, + "allocated_bytes_current": 69293369344, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145265893888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69293369344, + "active_bytes_peak": 145265893888, + "allocated_bytes_current": 69293369344, + "allocated_bytes_peak": 145265893888, + "device": 4, + "inactive_split_bytes_current": 9005897728, + "inactive_split_bytes_peak": 22530654720, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 148245577728, + "reserved_bytes_peak": 148245577728, + "segment_count_current": 1136, + "segment_count_peak": 1136, + "timestamp_utc": "2026-08-26T19:31:51.663704Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9005897728, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22530654720, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 148245577728, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148245577728, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1136, + "segment_count_current_delta": 0, + "segment_count_peak": 1136, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T193152432189Z.pickle", + "timestamp_utc": "2026-08-26T19:31:52.513219Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T193152432189Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T193152432189Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3d29a60ef10b227b7836cdc437bfe998b5f955b1 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T193152432189Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ea177e28d28133b2e4e45f63c51d06195532f114d5d09d4a9ac27c48b49d639 +size 4019716 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T193153293266Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T193153293266Z.json new file mode 100644 index 0000000000000000000000000000000000000000..03256aee19155056d142b815db4bcc427053501f --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T193153293266Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69293369344, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145265893888, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69293369344, + "active_bytes_peak": 145265893888, + "allocated_bytes_current": 69293369344, + "allocated_bytes_peak": 145265893888, + "device": 4, + "inactive_split_bytes_current": 9005897728, + "inactive_split_bytes_peak": 22530654720, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 148245577728, + "reserved_bytes_peak": 148245577728, + "segment_count_current": 1136, + "segment_count_peak": 1136, + "timestamp_utc": "2026-08-26T19:31:53.292845Z", + "world_size": 8 + }, + "allocated_bytes_current": 69293369344, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145265893888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69293369344, + "active_bytes_peak": 145265893888, + "allocated_bytes_current": 69293369344, + "allocated_bytes_peak": 145265893888, + "device": 4, + "inactive_split_bytes_current": 9005897728, + "inactive_split_bytes_peak": 22530654720, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 148245577728, + "reserved_bytes_peak": 148245577728, + "segment_count_current": 1136, + "segment_count_peak": 1136, + "timestamp_utc": "2026-08-26T19:31:52.519707Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9005897728, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22530654720, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 148245577728, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148245577728, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1136, + "segment_count_current_delta": 0, + "segment_count_peak": 1136, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T193153293266Z.pickle", + "timestamp_utc": "2026-08-26T19:31:53.375827Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T193153293266Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T193153293266Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..012f7efe61ba38a4ce4fb7458ece662381403078 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T193153293266Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcfbee5d15b474ea619c822c860e47dbcd080851682f8d3fb6bcadf3a231cbf0 +size 4019796 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T193154154357Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T193154154357Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0a5bf5e05500cab25eddc991327a30b5bb70720f --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T193154154357Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69293369344, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145265893888, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69293369344, + "active_bytes_peak": 145265893888, + "allocated_bytes_current": 69293369344, + "allocated_bytes_peak": 145265893888, + "device": 4, + "inactive_split_bytes_current": 9005897728, + "inactive_split_bytes_peak": 22530654720, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 148245577728, + "reserved_bytes_peak": 148245577728, + "segment_count_current": 1136, + "segment_count_peak": 1136, + "timestamp_utc": "2026-08-26T19:31:54.153923Z", + "world_size": 8 + }, + "allocated_bytes_current": 69293369344, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145265893888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69293369344, + "active_bytes_peak": 145265893888, + "allocated_bytes_current": 69293369344, + "allocated_bytes_peak": 145265893888, + "device": 4, + "inactive_split_bytes_current": 9005897728, + "inactive_split_bytes_peak": 22530654720, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 148245577728, + "reserved_bytes_peak": 148245577728, + "segment_count_current": 1136, + "segment_count_peak": 1136, + "timestamp_utc": "2026-08-26T19:31:53.382314Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9005897728, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22530654720, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 148245577728, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148245577728, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1136, + "segment_count_current_delta": 0, + "segment_count_peak": 1136, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T193154154357Z.pickle", + "timestamp_utc": "2026-08-26T19:31:54.239137Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T193154154357Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T193154154357Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..abbb6ca15f1c3d0f20d263275a82559f78fd47ce --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T193154154357Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2341bfa32076e2993bbef130320b1d1893dc9853db8fff6de1dc42307aeb251 +size 4019619 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T193150674643Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T193150674643Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0aec6cddf871e780b293754463d46e6553bb38d8 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T193150674643Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69300057088, + "active_bytes_current_delta": 45487180800, + "active_bytes_peak": 131036776960, + "active_bytes_peak_delta": 107211317760, + "after": { + "active_bytes_current": 69300057088, + "active_bytes_peak": 131036776960, + "allocated_bytes_current": 69300057088, + "allocated_bytes_peak": 131036776960, + "device": 5, + "inactive_split_bytes_current": 8982432768, + "inactive_split_bytes_peak": 22502780928, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 133680857088, + "reserved_bytes_peak": 133680857088, + "segment_count_current": 1125, + "segment_count_peak": 1125, + "timestamp_utc": "2026-08-26T19:31:50.674119Z", + "world_size": 8 + }, + "allocated_bytes_current": 69300057088, + "allocated_bytes_current_delta": 45487180800, + "allocated_bytes_peak": 131036776960, + "allocated_bytes_peak_delta": 107211317760, + "before": { + "active_bytes_current": 23812876288, + "active_bytes_peak": 23825459200, + "allocated_bytes_current": 23812876288, + "allocated_bytes_peak": 23825459200, + "device": 5, + "inactive_split_bytes_current": 807688192, + "inactive_split_bytes_peak": 1283983872, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 24639438848, + "reserved_bytes_peak": 24639438848, + "segment_count_current": 477, + "segment_count_peak": 477, + "timestamp_utc": "2026-08-26T19:31:27.995251Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8982432768, + "inactive_split_bytes_current_delta": 8174744576, + "inactive_split_bytes_peak": 22502780928, + "inactive_split_bytes_peak_delta": 21218797056, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 133680857088, + "reserved_bytes_current_delta": 109041418240, + "reserved_bytes_peak": 133680857088, + "reserved_bytes_peak_delta": 109041418240, + "segment_count_current": 1125, + "segment_count_current_delta": 648, + "segment_count_peak": 1125, + "segment_count_peak_delta": 648, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T193150674643Z.pickle", + "timestamp_utc": "2026-08-26T19:31:50.785321Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T193150674643Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T193150674643Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..89adb311b4758dbc3a0fa97c362c9530653db792 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T193150674643Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dce01fb88d3b598a1aa203d624a01967804d6b0815df1bcdfe49915ecc4e975 +size 5168331 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T193151565872Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T193151565872Z.json new file mode 100644 index 0000000000000000000000000000000000000000..80e0c507dd5b66f85212f11b35a6eb5e86fb03c6 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T193151565872Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69300057088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145260919296, + "active_bytes_peak_delta": 14224142336, + "after": { + "active_bytes_current": 69300057088, + "active_bytes_peak": 145260919296, + "allocated_bytes_current": 69300057088, + "allocated_bytes_peak": 145260919296, + "device": 5, + "inactive_split_bytes_current": 8982432768, + "inactive_split_bytes_peak": 22502780928, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 148220411904, + "reserved_bytes_peak": 148220411904, + "segment_count_current": 1135, + "segment_count_peak": 1135, + "timestamp_utc": "2026-08-26T19:31:51.565445Z", + "world_size": 8 + }, + "allocated_bytes_current": 69300057088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145260919296, + "allocated_bytes_peak_delta": 14224142336, + "before": { + "active_bytes_current": 69300057088, + "active_bytes_peak": 131036776960, + "allocated_bytes_current": 69300057088, + "allocated_bytes_peak": 131036776960, + "device": 5, + "inactive_split_bytes_current": 8982432768, + "inactive_split_bytes_peak": 22502780928, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 133680857088, + "reserved_bytes_peak": 133680857088, + "segment_count_current": 1125, + "segment_count_peak": 1125, + "timestamp_utc": "2026-08-26T19:31:50.794541Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8982432768, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22502780928, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 148220411904, + "reserved_bytes_current_delta": 14539554816, + "reserved_bytes_peak": 148220411904, + "reserved_bytes_peak_delta": 14539554816, + "segment_count_current": 1135, + "segment_count_current_delta": 10, + "segment_count_peak": 1135, + "segment_count_peak_delta": 10, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T193151565872Z.pickle", + "timestamp_utc": "2026-08-26T19:31:51.659872Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T193151565872Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T193151565872Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d1424d72234986aef6b6b325bb18294f5abeadac --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T193151565872Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b289a999c13888fd4a9def41946aee09b4f4d6b4089de62ef0a986f55a83e4a2 +size 4434838 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193152432023Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193152432023Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4cb98248c34bc78050f75b54ab259e4f42a3c676 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193152432023Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69300057088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145263933952, + "active_bytes_peak_delta": 3014656, + "after": { + "active_bytes_current": 69300057088, + "active_bytes_peak": 145263933952, + "allocated_bytes_current": 69300057088, + "allocated_bytes_peak": 145263933952, + "device": 5, + "inactive_split_bytes_current": 8982432768, + "inactive_split_bytes_peak": 22502780928, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 148220411904, + "reserved_bytes_peak": 148220411904, + "segment_count_current": 1135, + "segment_count_peak": 1135, + "timestamp_utc": "2026-08-26T19:31:52.431623Z", + "world_size": 8 + }, + "allocated_bytes_current": 69300057088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145263933952, + "allocated_bytes_peak_delta": 3014656, + "before": { + "active_bytes_current": 69300057088, + "active_bytes_peak": 145260919296, + "allocated_bytes_current": 69300057088, + "allocated_bytes_peak": 145260919296, + "device": 5, + "inactive_split_bytes_current": 8982432768, + "inactive_split_bytes_peak": 22502780928, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 148220411904, + "reserved_bytes_peak": 148220411904, + "segment_count_current": 1135, + "segment_count_peak": 1135, + "timestamp_utc": "2026-08-26T19:31:51.667400Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8982432768, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22502780928, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 148220411904, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148220411904, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1135, + "segment_count_current_delta": 0, + "segment_count_peak": 1135, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T193152432023Z.pickle", + "timestamp_utc": "2026-08-26T19:31:52.518777Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193152432023Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193152432023Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0f6ba6f4ac1f50b8596b82641c68203381545d9d --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T193152432023Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:475a40a1a96482c121f3e548017f47bcad35ddde50cf8b3cd9972a7925aa6c3c +size 4435015 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193153293164Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193153293164Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a0af9b706cefb16d228ac2f69d247f887f74f5fa --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193153293164Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69300057088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145263933952, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69300057088, + "active_bytes_peak": 145263933952, + "allocated_bytes_current": 69300057088, + "allocated_bytes_peak": 145263933952, + "device": 5, + "inactive_split_bytes_current": 8982432768, + "inactive_split_bytes_peak": 22502780928, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 148220411904, + "reserved_bytes_peak": 148220411904, + "segment_count_current": 1135, + "segment_count_peak": 1135, + "timestamp_utc": "2026-08-26T19:31:53.292755Z", + "world_size": 8 + }, + "allocated_bytes_current": 69300057088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145263933952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69300057088, + "active_bytes_peak": 145263933952, + "allocated_bytes_current": 69300057088, + "allocated_bytes_peak": 145263933952, + "device": 5, + "inactive_split_bytes_current": 8982432768, + "inactive_split_bytes_peak": 22502780928, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 148220411904, + "reserved_bytes_peak": 148220411904, + "segment_count_current": 1135, + "segment_count_peak": 1135, + "timestamp_utc": "2026-08-26T19:31:52.525773Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8982432768, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22502780928, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 148220411904, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148220411904, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1135, + "segment_count_current_delta": 0, + "segment_count_peak": 1135, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T193153293164Z.pickle", + "timestamp_utc": "2026-08-26T19:31:53.379736Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193153293164Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193153293164Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..77dd42d059cc86abb21e980e4a8f30fb34acda20 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T193153293164Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:124950402f42739427a8b960e82d0a8701c91f9394d76a7048da37e55d55cd8e +size 4435012 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193154154054Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193154154054Z.json new file mode 100644 index 0000000000000000000000000000000000000000..208b3eb3517e2079c3347a305627a7afcb6a2a1a --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193154154054Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69300057088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145263933952, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69300057088, + "active_bytes_peak": 145263933952, + "allocated_bytes_current": 69300057088, + "allocated_bytes_peak": 145263933952, + "device": 5, + "inactive_split_bytes_current": 8982432768, + "inactive_split_bytes_peak": 22502780928, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 148220411904, + "reserved_bytes_peak": 148220411904, + "segment_count_current": 1135, + "segment_count_peak": 1135, + "timestamp_utc": "2026-08-26T19:31:54.153650Z", + "world_size": 8 + }, + "allocated_bytes_current": 69300057088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145263933952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69300057088, + "active_bytes_peak": 145263933952, + "allocated_bytes_current": 69300057088, + "allocated_bytes_peak": 145263933952, + "device": 5, + "inactive_split_bytes_current": 8982432768, + "inactive_split_bytes_peak": 22502780928, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 148220411904, + "reserved_bytes_peak": 148220411904, + "segment_count_current": 1135, + "segment_count_peak": 1135, + "timestamp_utc": "2026-08-26T19:31:53.386744Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8982432768, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22502780928, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 148220411904, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148220411904, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1135, + "segment_count_current_delta": 0, + "segment_count_peak": 1135, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T193154154054Z.pickle", + "timestamp_utc": "2026-08-26T19:31:54.241848Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193154154054Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193154154054Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bbdd01986ffa87ee5f1a1089b7d74d7170fe0e53 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T193154154054Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc6a4bfa9870dc458a63d70828cee74ba6b3cc2a074e5ea4a5e20c078f735d82 +size 4435123 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193150674629Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193150674629Z.json new file mode 100644 index 0000000000000000000000000000000000000000..037dd00bd7692dfb25e1c03e10d78327e12b3943 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193150674629Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69289378304, + "active_bytes_current_delta": 45495094784, + "active_bytes_peak": 131019717632, + "active_bytes_peak_delta": 107212851200, + "after": { + "active_bytes_current": 69289378304, + "active_bytes_peak": 131019717632, + "allocated_bytes_current": 69289378304, + "allocated_bytes_peak": 131019717632, + "device": 6, + "inactive_split_bytes_current": 8980528640, + "inactive_split_bytes_peak": 22506885632, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 134496649216, + "reserved_bytes_peak": 134496649216, + "segment_count_current": 1122, + "segment_count_peak": 1122, + "timestamp_utc": "2026-08-26T19:31:50.674098Z", + "world_size": 8 + }, + "allocated_bytes_current": 69289378304, + "allocated_bytes_current_delta": 45495094784, + "allocated_bytes_peak": 131019717632, + "allocated_bytes_peak_delta": 107212851200, + "before": { + "active_bytes_current": 23794283520, + "active_bytes_peak": 23806866432, + "allocated_bytes_current": 23794283520, + "allocated_bytes_peak": 23806866432, + "device": 6, + "inactive_split_bytes_current": 815795200, + "inactive_split_bytes_peak": 1283983872, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 24626855936, + "reserved_bytes_peak": 24626855936, + "segment_count_current": 471, + "segment_count_peak": 471, + "timestamp_utc": "2026-08-26T19:31:27.995008Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8980528640, + "inactive_split_bytes_current_delta": 8164733440, + "inactive_split_bytes_peak": 22506885632, + "inactive_split_bytes_peak_delta": 21222901760, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 134496649216, + "reserved_bytes_current_delta": 109869793280, + "reserved_bytes_peak": 134496649216, + "reserved_bytes_peak_delta": 109869793280, + "segment_count_current": 1122, + "segment_count_current_delta": 651, + "segment_count_peak": 1122, + "segment_count_peak_delta": 651, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T193150674629Z.pickle", + "timestamp_utc": "2026-08-26T19:31:50.780441Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193150674629Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193150674629Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..83fbb76f6ea580df24edc3e11a9c9467044cc2ff --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T193150674629Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f88361557ab6e1b9c8ae297d5ab89b0a7f20c0b374728e6d9d5b005bfffdd447 +size 4754053 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193151565834Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193151565834Z.json new file mode 100644 index 0000000000000000000000000000000000000000..941bca2ef00be44238d6f01181c7349ca348b398 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193151565834Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69289378304, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145246516736, + "active_bytes_peak_delta": 14226799104, + "after": { + "active_bytes_current": 69289378304, + "active_bytes_peak": 145246516736, + "allocated_bytes_current": 69289378304, + "allocated_bytes_peak": 145246516736, + "device": 6, + "inactive_split_bytes_current": 8980528640, + "inactive_split_bytes_peak": 22506885632, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 147392036864, + "reserved_bytes_peak": 147392036864, + "segment_count_current": 1133, + "segment_count_peak": 1133, + "timestamp_utc": "2026-08-26T19:31:51.565425Z", + "world_size": 8 + }, + "allocated_bytes_current": 69289378304, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145246516736, + "allocated_bytes_peak_delta": 14226799104, + "before": { + "active_bytes_current": 69289378304, + "active_bytes_peak": 131019717632, + "allocated_bytes_current": 69289378304, + "allocated_bytes_peak": 131019717632, + "device": 6, + "inactive_split_bytes_current": 8980528640, + "inactive_split_bytes_peak": 22506885632, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 134496649216, + "reserved_bytes_peak": 134496649216, + "segment_count_current": 1122, + "segment_count_peak": 1122, + "timestamp_utc": "2026-08-26T19:31:50.788419Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8980528640, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22506885632, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 147392036864, + "reserved_bytes_current_delta": 12895387648, + "reserved_bytes_peak": 147392036864, + "reserved_bytes_peak_delta": 12895387648, + "segment_count_current": 1133, + "segment_count_current_delta": 11, + "segment_count_peak": 1133, + "segment_count_peak_delta": 11, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T193151565834Z.pickle", + "timestamp_utc": "2026-08-26T19:31:51.653708Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193151565834Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193151565834Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c28857edc188a0032789d8ea286b31740f3f5378 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T193151565834Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:876abb9e7781f4901f94aad1f185c6d4e1634bb073e677583f1cafb49a90f991 +size 4015051 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193152432042Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193152432042Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cf168f961b1d664e5c4e0f2466fc548d96ef1400 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193152432042Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69289378304, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145246516736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69289378304, + "active_bytes_peak": 145246516736, + "allocated_bytes_current": 69289378304, + "allocated_bytes_peak": 145246516736, + "device": 6, + "inactive_split_bytes_current": 8980528640, + "inactive_split_bytes_peak": 22506885632, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 147392036864, + "reserved_bytes_peak": 147392036864, + "segment_count_current": 1133, + "segment_count_peak": 1133, + "timestamp_utc": "2026-08-26T19:31:52.431629Z", + "world_size": 8 + }, + "allocated_bytes_current": 69289378304, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145246516736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69289378304, + "active_bytes_peak": 145246516736, + "allocated_bytes_current": 69289378304, + "allocated_bytes_peak": 145246516736, + "device": 6, + "inactive_split_bytes_current": 8980528640, + "inactive_split_bytes_peak": 22506885632, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 147392036864, + "reserved_bytes_peak": 147392036864, + "segment_count_current": 1133, + "segment_count_peak": 1133, + "timestamp_utc": "2026-08-26T19:31:51.660116Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8980528640, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22506885632, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 147392036864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 147392036864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1133, + "segment_count_current_delta": 0, + "segment_count_peak": 1133, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T193152432042Z.pickle", + "timestamp_utc": "2026-08-26T19:31:52.511175Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193152432042Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193152432042Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fb7a4c4c79bfcc4fdad67d571429b248c16cc1ab --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T193152432042Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76410826bee769d0ba261a4cfea3eb28b743d338146ca95f4cff0e92f233d8ec +size 4014977 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193153293157Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193153293157Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ed99542c4289f8feabc09eec5fa49ccf2c598dab --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193153293157Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69289378304, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145246516736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69289378304, + "active_bytes_peak": 145246516736, + "allocated_bytes_current": 69289378304, + "allocated_bytes_peak": 145246516736, + "device": 6, + "inactive_split_bytes_current": 8980528640, + "inactive_split_bytes_peak": 22506885632, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 147392036864, + "reserved_bytes_peak": 147392036864, + "segment_count_current": 1133, + "segment_count_peak": 1133, + "timestamp_utc": "2026-08-26T19:31:53.292766Z", + "world_size": 8 + }, + "allocated_bytes_current": 69289378304, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145246516736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69289378304, + "active_bytes_peak": 145246516736, + "allocated_bytes_current": 69289378304, + "allocated_bytes_peak": 145246516736, + "device": 6, + "inactive_split_bytes_current": 8980528640, + "inactive_split_bytes_peak": 22506885632, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 147392036864, + "reserved_bytes_peak": 147392036864, + "segment_count_current": 1133, + "segment_count_peak": 1133, + "timestamp_utc": "2026-08-26T19:31:52.517378Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8980528640, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22506885632, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 147392036864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 147392036864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1133, + "segment_count_current_delta": 0, + "segment_count_peak": 1133, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T193153293157Z.pickle", + "timestamp_utc": "2026-08-26T19:31:53.373451Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193153293157Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193153293157Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4065c7198cbfb6827ac21884a3e61651a601f9fa --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T193153293157Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05d752ad75ab97e16400a5c7db8ff48bf84a094f2d426048687361152080eb48 +size 4015107 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193154154066Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193154154066Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d0e3b1a2bbc8d6926040ca79b3577ae9c84485c3 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193154154066Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69289378304, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145246516736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69289378304, + "active_bytes_peak": 145246516736, + "allocated_bytes_current": 69289378304, + "allocated_bytes_peak": 145246516736, + "device": 6, + "inactive_split_bytes_current": 8980528640, + "inactive_split_bytes_peak": 22506885632, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 147392036864, + "reserved_bytes_peak": 147392036864, + "segment_count_current": 1133, + "segment_count_peak": 1133, + "timestamp_utc": "2026-08-26T19:31:54.153670Z", + "world_size": 8 + }, + "allocated_bytes_current": 69289378304, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145246516736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69289378304, + "active_bytes_peak": 145246516736, + "allocated_bytes_current": 69289378304, + "allocated_bytes_peak": 145246516736, + "device": 6, + "inactive_split_bytes_current": 8980528640, + "inactive_split_bytes_peak": 22506885632, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 147392036864, + "reserved_bytes_peak": 147392036864, + "segment_count_current": 1133, + "segment_count_peak": 1133, + "timestamp_utc": "2026-08-26T19:31:53.379653Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8980528640, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22506885632, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 147392036864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 147392036864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1133, + "segment_count_current_delta": 0, + "segment_count_peak": 1133, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T193154154066Z.pickle", + "timestamp_utc": "2026-08-26T19:31:54.234249Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193154154066Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193154154066Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..12b28dc8328fb6970b315bfa9f59b23099f48878 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T193154154066Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6202a9849a213bbc4215fd86a0421f72640f8b3cc9725ebd9fc4b32c0b2deb0 +size 4014978 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193150674644Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193150674644Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9c082ba344a74ef1f861cec15190f3b88f5fc706 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193150674644Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69293543936, + "active_bytes_current_delta": 45490263552, + "active_bytes_peak": 131036168704, + "active_bytes_peak_delta": 107220305408, + "after": { + "active_bytes_current": 69293543936, + "active_bytes_peak": 131036168704, + "allocated_bytes_current": 69293543936, + "allocated_bytes_peak": 131036168704, + "device": 7, + "inactive_split_bytes_current": 8995237376, + "inactive_split_bytes_peak": 22446469120, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 133687148544, + "reserved_bytes_peak": 133687148544, + "segment_count_current": 1127, + "segment_count_peak": 1127, + "timestamp_utc": "2026-08-26T19:31:50.674115Z", + "world_size": 8 + }, + "allocated_bytes_current": 69293543936, + "allocated_bytes_current_delta": 45490263552, + "allocated_bytes_peak": 131036168704, + "allocated_bytes_peak_delta": 107220305408, + "before": { + "active_bytes_current": 23803280384, + "active_bytes_peak": 23815863296, + "allocated_bytes_current": 23803280384, + "allocated_bytes_peak": 23815863296, + "device": 7, + "inactive_split_bytes_current": 890684416, + "inactive_split_bytes_peak": 1283983872, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 24712839168, + "reserved_bytes_peak": 24712839168, + "segment_count_current": 477, + "segment_count_peak": 477, + "timestamp_utc": "2026-08-26T19:31:27.995210Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8995237376, + "inactive_split_bytes_current_delta": 8104552960, + "inactive_split_bytes_peak": 22446469120, + "inactive_split_bytes_peak_delta": 21162485248, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 133687148544, + "reserved_bytes_current_delta": 108974309376, + "reserved_bytes_peak": 133687148544, + "reserved_bytes_peak_delta": 108974309376, + "segment_count_current": 1127, + "segment_count_current_delta": 650, + "segment_count_peak": 1127, + "segment_count_peak_delta": 650, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T193150674644Z.pickle", + "timestamp_utc": "2026-08-26T19:31:50.785385Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193150674644Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193150674644Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1d76175cc7b01eb540405e777fdf9ad22d9ee8dc --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T193150674644Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0827bb0621596719fad37c44bdf657c1af5c2d614fa75cd5faf2119542e1b8e2 +size 4744128 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193151565891Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193151565891Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6094e6d95a5e5b336ca99bc5c0006f7c582f29ca --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193151565891Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69293543936, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145262150144, + "active_bytes_peak_delta": 14225981440, + "after": { + "active_bytes_current": 69293543936, + "active_bytes_peak": 145262150144, + "allocated_bytes_current": 69293543936, + "allocated_bytes_peak": 145262150144, + "device": 7, + "inactive_split_bytes_current": 8995237376, + "inactive_split_bytes_peak": 22446469120, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:31:51.565461Z", + "world_size": 8 + }, + "allocated_bytes_current": 69293543936, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145262150144, + "allocated_bytes_peak_delta": 14225981440, + "before": { + "active_bytes_current": 69293543936, + "active_bytes_peak": 131036168704, + "allocated_bytes_current": 69293543936, + "allocated_bytes_peak": 131036168704, + "device": 7, + "inactive_split_bytes_current": 8995237376, + "inactive_split_bytes_peak": 22446469120, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 133687148544, + "reserved_bytes_peak": 133687148544, + "segment_count_current": 1127, + "segment_count_peak": 1127, + "timestamp_utc": "2026-08-26T19:31:50.794534Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8995237376, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22446469120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 148226703360, + "reserved_bytes_current_delta": 14539554816, + "reserved_bytes_peak": 148226703360, + "reserved_bytes_peak_delta": 14539554816, + "segment_count_current": 1137, + "segment_count_current_delta": 10, + "segment_count_peak": 1137, + "segment_count_peak_delta": 10, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T193151565891Z.pickle", + "timestamp_utc": "2026-08-26T19:31:51.660611Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193151565891Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193151565891Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..717f60f4e1fc14073d69dbaf2ae9da2fa7beeb9b --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T193151565891Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:484afdc5be8a6e8d892888601f47da33cac444c9dc1a819d9137952d3464c897 +size 4010750 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193152432054Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193152432054Z.json new file mode 100644 index 0000000000000000000000000000000000000000..78413da6d7f1735fc604aed93dcb91e9bd55dcdc --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193152432054Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69293543936, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145262150144, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69293543936, + "active_bytes_peak": 145262150144, + "allocated_bytes_current": 69293543936, + "allocated_bytes_peak": 145262150144, + "device": 7, + "inactive_split_bytes_current": 8995237376, + "inactive_split_bytes_peak": 22446469120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:31:52.431642Z", + "world_size": 8 + }, + "allocated_bytes_current": 69293543936, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145262150144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69293543936, + "active_bytes_peak": 145262150144, + "allocated_bytes_current": 69293543936, + "allocated_bytes_peak": 145262150144, + "device": 7, + "inactive_split_bytes_current": 8995237376, + "inactive_split_bytes_peak": 22446469120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:31:51.667949Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8995237376, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22446469120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 148226703360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148226703360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1137, + "segment_count_current_delta": 0, + "segment_count_peak": 1137, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T193152432054Z.pickle", + "timestamp_utc": "2026-08-26T19:31:52.522712Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193152432054Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193152432054Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fb3b1c3393bf3daafdf9d72b233e0b6f670ea663 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T193152432054Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f43ce8c9e6c104d23a003959e84b6921c074202367bfa38f5c8a66bf480352f +size 4011044 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193153293160Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193153293160Z.json new file mode 100644 index 0000000000000000000000000000000000000000..834c60116cddd02670647b0df0a503b8a2e93efa --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193153293160Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69293543936, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145262150144, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69293543936, + "active_bytes_peak": 145262150144, + "allocated_bytes_current": 69293543936, + "allocated_bytes_peak": 145262150144, + "device": 7, + "inactive_split_bytes_current": 8995237376, + "inactive_split_bytes_peak": 22446469120, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:31:53.292757Z", + "world_size": 8 + }, + "allocated_bytes_current": 69293543936, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145262150144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69293543936, + "active_bytes_peak": 145262150144, + "allocated_bytes_current": 69293543936, + "allocated_bytes_peak": 145262150144, + "device": 7, + "inactive_split_bytes_current": 8995237376, + "inactive_split_bytes_peak": 22446469120, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:31:52.530232Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8995237376, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22446469120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 148226703360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148226703360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1137, + "segment_count_current_delta": 0, + "segment_count_peak": 1137, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T193153293160Z.pickle", + "timestamp_utc": "2026-08-26T19:31:53.381215Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193153293160Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193153293160Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..39b98d6c0f538f6e9d4ee3269e5d1d540a2e3943 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T193153293160Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7790fb20bacca82ae7d131f2cb71c1fd6820afa30bb0140ac05b37ceccc2768 +size 4011042 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193154154066Z.json b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193154154066Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0e252563350a90a54cfcfabf60cfc58adde6775f --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193154154066Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 69293543936, + "active_bytes_current_delta": 0, + "active_bytes_peak": 145262150144, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 69293543936, + "active_bytes_peak": 145262150144, + "allocated_bytes_current": 69293543936, + "allocated_bytes_peak": 145262150144, + "device": 7, + "inactive_split_bytes_current": 8995237376, + "inactive_split_bytes_peak": 22446469120, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:31:54.153656Z", + "world_size": 8 + }, + "allocated_bytes_current": 69293543936, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 145262150144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 69293543936, + "active_bytes_peak": 145262150144, + "allocated_bytes_current": 69293543936, + "allocated_bytes_peak": 145262150144, + "device": 7, + "inactive_split_bytes_current": 8995237376, + "inactive_split_bytes_peak": 22446469120, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 148226703360, + "reserved_bytes_peak": 148226703360, + "segment_count_current": 1137, + "segment_count_peak": 1137, + "timestamp_utc": "2026-08-26T19:31:53.388306Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8995237376, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22446469120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 148226703360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148226703360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 1137, + "segment_count_current_delta": 0, + "segment_count_peak": 1137, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T193154154066Z.pickle", + "timestamp_utc": "2026-08-26T19:31:54.244763Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193154154066Z.pickle b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193154154066Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0e9df38f43f45a53a4f269c2111666193c54e893 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T193154154066Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d57747932bf01a12337bf8fd221c282e54dcb6f5e38dc77f114a46a194f3caaf +size 4011084 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/run.env b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..47eaa8e2cb055ec8a90005449802330f848ccdc5 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/run.env @@ -0,0 +1,61 @@ +backend=moonep +config=DSV2_LITE_L24 +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=moonep_backend_better_memory_dump_support +commit=453b84c7 +num_layers=24 +hidden=2048 +ffn_hidden=10944 +moe_ffn_hidden=1408 +num_experts=64 +topk=6 +experts_per_rank=8 +expert_params=8650752 +heads=16 +kv_groups=0 +kv_channels=128 +qk_layernorm=1 +shared_ffn=2816 +shared_expert_overlap=0 +mla=1 +dense_first_layers=1 +moe_layers=23 +rotary_base=10000 +rotary_scaling=40 +qkv_bias=0 +router_balancing=seq_aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=49152 +mean_tokens_per_expert=6144 +slot_policy=matched +slots_per_rank=8 +capacity_mode=none +capacity=none +cuda_graph=0 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=5 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/train.log b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..8d66b6eaa539ff50a940481f04c2eac343eee680 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/train.log @@ -0,0 +1,1428 @@ +============================================================== + DSV2_LITE_L24 / moonep (slots/rank=8 policy=matched) + E=64 K=6 H'=1408 m=8 + L=24 seq=8192 gbs=8 (1 microbatches) + cap_mode=none cap=none cg=0 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 453b84c7 +============================================================== +W0826 19:30:59.855000 681961 torch/distributed/run.py:766] +W0826 19:30:59.855000 681961 torch/distributed/run.py:766] ***************************************** +W0826 19:30:59.855000 681961 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 19:30:59.855000 681961 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 24 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 10944 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ /workspace/tierb_study_20260826_144644/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1/memory_snapshots/snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. False + moe_enable_moonep ............................... True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1408 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_moonep_num_sms .............................. None + moe_num_echo_experts ............................ None + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 6 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 2816 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 0.707 + mscale_all_dim .................................. 0.707 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 24 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_cuda_graph_memory_history ................ False + record_full_program_memory_history .............. False + record_iteration_memory_history ................. True + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 40.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 5 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 77 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank2]:[W826 19:31:18.626551651 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 19:31:18.626668799 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 19:31:18.626681406 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 19:31:18.626741462 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 19:31:18.626782800 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 19:31:18.626892660 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 19:31:18.635839298 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.033 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 19:31:18.786012951 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.453 seconds +time to initialize megatron (seconds): 6.673 +[after megatron is initialized] datetime: 2026-08-26 19:31:24 +building GPT model ... +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1531: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1531: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1531: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1531: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1531: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1531: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1531: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1531: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 2596386816 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (48503296 elements, 48503296 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.23.self_attention.linear_kv_down_proj.weight + module.decoder.layers.23.self_attention.linear_q_proj.weight + module.decoder.layers.23.input_layernorm.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight +Params for bucket 3 (40901632 elements, 40901632 padded size): + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.22.self_attention.linear_q_proj.weight + module.decoder.layers.22.input_layernorm.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.21.self_attention.linear_kv_down_proj.weight + module.decoder.layers.21.self_attention.linear_q_proj.weight +Params for bucket 4 (41163264 elements, 41163264 padded size): + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.linear_q_proj.weight + module.decoder.layers.20.input_layernorm.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.21.input_layernorm.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.20.self_attention.linear_kv_down_proj.weight + module.decoder.layers.20.self_attention.linear_proj.weight +Params for bucket 5 (42734080 elements, 42734080 padded size): + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.self_attention.linear_kv_down_proj.weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.self_attention.linear_q_proj.weight + module.decoder.layers.19.input_layernorm.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight +Params for bucket 6 (40901632 elements, 40901632 padded size): + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.self_attention.linear_kv_down_proj.weight + module.decoder.layers.17.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.18.input_layernorm.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.pre_mlp_layernorm.weight +Params for bucket 7 (41163264 elements, 41163264 padded size): + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.input_layernorm.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.16.self_attention.linear_kv_down_proj.weight + module.decoder.layers.16.self_attention.linear_q_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.16.input_layernorm.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.pre_mlp_layernorm.weight +Params for bucket 8 (42734080 elements, 42734080 padded size): + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_q_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.weight + module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.15.input_layernorm.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight +Params for bucket 9 (40901632 elements, 40901632 padded size): + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.14.self_attention.linear_q_proj.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.13.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.input_layernorm.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight +Params for bucket 10 (41163264 elements, 41163264 padded size): + module.decoder.layers.13.input_layernorm.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.12.self_attention.linear_kv_down_proj.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_q_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.input_layernorm.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight +Params for bucket 11 (42734080 elements, 42734080 padded size): + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_q_proj.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.input_layernorm.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight + module.decoder.layers.11.self_attention.linear_kv_down_proj.weight +Params for bucket 12 (40901632 elements, 40901632 padded size): + module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.10.self_attention.linear_q_proj.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.input_layernorm.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.9.self_attention.linear_q_proj.weight +Params for bucket 13 (41163264 elements, 41163264 padded size): + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.input_layernorm.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.8.self_attention.linear_kv_down_proj.weight + module.decoder.layers.8.self_attention.linear_q_proj.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.input_layernorm.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight +Params for bucket 14 (42734080 elements, 42734080 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.self_attention.linear_kv_down_proj.weight + module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.input_layernorm.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.mlp.router.weight +Params for bucket 15 (40901632 elements, 40901632 padded size): + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_q_proj.weight + module.decoder.layers.6.input_layernorm.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.5.self_attention.linear_q_proj.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_kv_down_proj.weight +Params for bucket 16 (41163264 elements, 41163264 padded size): + module.decoder.layers.5.input_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_q_proj.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.input_layernorm.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.4.self_attention.linear_kv_down_proj.weight +Params for bucket 17 (42734080 elements, 42734080 padded size): + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.self_attention.linear_kv_down_proj.weight + module.decoder.layers.3.self_attention.linear_q_proj.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.input_layernorm.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight +Params for bucket 18 (40901632 elements, 40901632 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_q_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.weight + module.decoder.layers.2.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.2.self_attention.linear_q_proj.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.input_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight +Params for bucket 19 (71436288 elements, 71436288 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc1.weight + module.decoder.layers.0.mlp.linear_fc2.weight +Params for bucket 20 (116789760 elements, 116789760 padded size): + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.input_layernorm.weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_q_proj.weight + module.decoder.layers.0.self_attention.linear_proj.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 +Params for bucket 3 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 +Params for bucket 4 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 +Params for bucket 7 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 +Params for bucket 8 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 +Params for bucket 11 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 +Params for bucket 12 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 +Params for bucket 15 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 +Params for bucket 16 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 +Params for bucket 19 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 +Params for bucket 20 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 +Params for bucket 23 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 +Params for bucket 24 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 25 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 28 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 +Params for bucket 29 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 +Params for bucket 32 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 +Params for bucket 33 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 36 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 37 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 +Params for bucket 40 (17301504 elements, 17301504 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 19:31:27 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 40 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(40, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from ddd6755186c6b13339ab7bd4e0fae553-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from ddd6755186c6b13339ab7bd4e0fae553-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from ddd6755186c6b13339ab7bd4e0fae553-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 19:31:27 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3439.22, 3457.60) + train/valid/test-data-iterators-setup ..........: (63.21, 68.22) +[before the start of training step] datetime: 2026-08-26 19:31:27 +INFO:moonep.api:MoonEP comm buffer sizes: + config : R=8, S=8192, H=2048, E=64, K=6, NvS=51184, NvS_padded=51200 + sm_count : num_sms=32, num_sms_dedup=148 + hidden_buffer_size : 200.00 MiB + meta_buffer_size : 2.00 MiB + local_temp_buffer_size : 2.01 MiB + total_size : 204.01 MiB +WARNING:megatron.core.rerun_state_machine:Result validation enabled + [2026-08-26 19:31:50] iteration 1/ 5 | consumed samples: 8 | elapsed time per iteration (ms): 22679.5 | throughput per GPU (TFLOP/s/GPU): 5.6 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092206E+01 | loss scale: 1.0 | grad norm: 30.650 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 13.53 +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 13.74 +Number of parameters in most loaded shard in billions: 13.7356 +Theoretical memory footprints: weight and optimizer=98244.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 66079.91357421875 | max allocated: 124949.08837890625 | reserved: 127502.0 | max reserved: 127502.0 + [2026-08-26 19:31:51] iteration 2/ 5 | consumed samples: 16 | elapsed time per iteration (ms): 892.0 | throughput per GPU (TFLOP/s/GPU): 143.0 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.091986E+01 | loss scale: 1.0 | grad norm: 30.565 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:31:52] iteration 3/ 5 | consumed samples: 24 | elapsed time per iteration (ms): 866.1 | throughput per GPU (TFLOP/s/GPU): 147.3 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.956084E+00 | loss scale: 1.0 | grad norm: 42.533 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:31:53] iteration 4/ 5 | consumed samples: 32 | elapsed time per iteration (ms): 861.0 | throughput per GPU (TFLOP/s/GPU): 148.2 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.478545E+00 | loss scale: 1.0 | grad norm: 7.152 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 19:31:54] iteration 5/ 5 | consumed samples: 40 | elapsed time per iteration (ms): 860.7 | throughput per GPU (TFLOP/s/GPU): 148.2 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.091896E+01 | loss scale: 1.0 | grad norm: 452.732 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 19:31:54 +[rank0]:[W826 19:31:55.804208639 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 19:31:56.227576544 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 19:31:56.227617268 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 19:31:56.227659490 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 19:31:56.227701139 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 19:31:56.237806439 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 19:31:56.388584645 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 19:31:56.388636356 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 19:31:58.531000 681961 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 19:31:58.532000 681961 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 682051 closing signal SIGTERM +W0826 19:31:58.532000 681961 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 682052 closing signal SIGTERM +W0826 19:31:58.535000 681961 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 682053 closing signal SIGTERM +W0826 19:31:58.538000 681961 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 682054 closing signal SIGTERM +W0826 19:31:58.541000 681961 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 682055 closing signal SIGTERM +W0826 19:31:58.544000 681961 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 682056 closing signal SIGTERM +W0826 19:31:58.548000 681961 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 682057 closing signal SIGTERM +W0826 19:31:58.549000 681961 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 682058 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 681961 got signal: 15 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/mem_samples.csv b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..6bf40d9ef820de597b2e232fe6a21990a212293d --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/mem_samples.csv @@ -0,0 +1,16 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/run.env b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..fbb1b78b98bc8c731bc81c485fa8a425cc733e48 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/run.env @@ -0,0 +1,61 @@ +backend=moonep +config=QWEN2_57B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=moonep_backend_better_memory_dump_support +commit=453b84c7 +num_layers=10 +hidden=3584 +ffn_hidden=2560 +moe_ffn_hidden=2560 +num_experts=64 +topk=8 +experts_per_rank=8 +expert_params=27525120 +heads=28 +kv_groups=4 +kv_channels= +qk_layernorm=0 +shared_ffn=20480 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=10 +rotary_base=1000000 +rotary_scaling= +qkv_bias=1 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=8 +capacity_mode=none +capacity=none +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=memory +memory_dump_mode=iteration +memory_snapshot_path=/workspace/tierb_study_20260826_144644/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot.pickle +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=6 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/train.log b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..2d2be22c80874cb3b7a87a0ec6da53b55bd88b10 --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/train.log @@ -0,0 +1,12 @@ +============================================================== + QWEN2_57B / moonep (slots/rank=8 policy=matched) + E=64 K=8 H'=2560 m=8 + L=10 seq=4096 gbs=8 (1 microbatches) + cap_mode=none cap=none cg=1 recompute=none + memory_dump=iteration snapshots=/workspace/tierb_study_20260826_144644/moonep/tierbmem/tierbmem_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__memiteration__it6/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 453b84c7 +============================================================== +W0826 19:32:05.069000 686032 torch/distributed/run.py:766] +W0826 19:32:05.069000 686032 torch/distributed/run.py:766] ***************************************** +W0826 19:32:05.069000 686032 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 19:32:05.069000 686032 torch/distributed/run.py:766] ***************************************** diff --git a/moonep/tierbmem/tierbmem_gbs8/results.tsv b/moonep/tierbmem/tierbmem_gbs8/results.tsv new file mode 100644 index 0000000000000000000000000000000000000000..8e2688ff419e499998344d664078abeb0098c76d --- /dev/null +++ b/moonep/tierbmem/tierbmem_gbs8/results.tsv @@ -0,0 +1,3 @@ +config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir +DSV2_LITE_L24 moonep matched 8 8192 8 none none 1 OK 767.90 767.90 165295 /workspace/tierb_study_20260826_144644/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__memiteration__it6/rep1 +DSV2_LITE_L24 moonep matched 8 8192 8 none none 1 OK NA NA 157153 /workspace/tierb_study_20260826_144644/moonep/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__nocg__memiteration__it5/rep1