Upload out_slim/tiny_LLaMA_1bret_id

.gitattributes

@@ -35,3 +35,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 out_slim/tiny_LLaMA_1b_2k_ret_tr/wandb/run-20250207_154242-pj6ofwzx/run-pj6ofwzx.wandb filter=lfs diff=lfs merge=lfs -text
 out_slim/tiny_LLaMA_1b_2k_ret_tr/wandb/run-20250209_034737-r3mb820k/run-r3mb820k.wandb filter=lfs diff=lfs merge=lfs -text
+out_slim/tiny_LLaMA_1bret_id/wandb/run-20250214_152403-5aszxlzy/run-5aszxlzy.wandb filter=lfs diff=lfs merge=lfs -text

out_slim/tiny_LLaMA_1bret_id/step-000995-ckpt.txt

@@ -0,0 +1 @@
+104.62479939326758

out_slim/tiny_LLaMA_1bret_id/step-001989-ckpt.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d3c13aa65d78775afff7381bb5e4a8e68ea166c0bf7b464c1d09dafc2ac2f3e
+size 13200815114

out_slim/tiny_LLaMA_1bret_id/step-001990-ckpt.txt

@@ -0,0 +1 @@
+54.974257189735646

out_slim/tiny_LLaMA_1bret_id/step-002984-ckpt.txt

@@ -0,0 +1 @@
+42.39873271749793

out_slim/tiny_LLaMA_1bret_id/step-003978-ckpt.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc1e83c585955be44ed48502230c949ac05b5750d6a7a4c5fdad94919978ba12
+size 13200815114

out_slim/tiny_LLaMA_1bret_id/step-003979-ckpt.txt

@@ -0,0 +1 @@
+39.52086898941868

out_slim/tiny_LLaMA_1bret_id/step-004974-ckpt.txt

@@ -0,0 +1 @@
+33.492787854104414

out_slim/tiny_LLaMA_1bret_id/step-005967-ckpt.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:911f0517ea25d98582566d648ead5361e2c84304d02aaf49d2f49c03a683e710
+size 13200815114

out_slim/tiny_LLaMA_1bret_id/step-005969-ckpt.txt

@@ -0,0 +1 @@
+31.604771482279688

out_slim/tiny_LLaMA_1bret_id/step-006964-ckpt.txt

@@ -0,0 +1 @@
+29.359950545215376

out_slim/tiny_LLaMA_1bret_id/step-007956-ckpt.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a33252e572c55920735e9371f00aa1575cc88468fb4b7bfee6e89a279dd814c
+size 13200815114

out_slim/tiny_LLaMA_1bret_id/step-007958-ckpt.txt

@@ -0,0 +1 @@
+28.191391797859318

out_slim/tiny_LLaMA_1bret_id/step-008953-ckpt.txt

@@ -0,0 +1 @@
+26.502949056804884

out_slim/tiny_LLaMA_1bret_id/step-009945-ckpt.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d150ab768c894819b21dee0a010eb782455958f27f3747172c781d5c93023a0
+size 13200815114

out_slim/tiny_LLaMA_1bret_id/step-009948-ckpt.txt

@@ -0,0 +1 @@
+26.529400087824722

out_slim/tiny_LLaMA_1bret_id/step-010943-ckpt.txt

@@ -0,0 +1 @@
+26.16802611254661

out_slim/tiny_LLaMA_1bret_id/step-011934-ckpt.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6f588558c1642909449ac04d345c55a9af99188cc3376b7f2e6403112ac37cb
+size 13200815114

out_slim/tiny_LLaMA_1bret_id/step-011938-ckpt.txt

@@ -0,0 +1 @@
+25.28409984560651

out_slim/tiny_LLaMA_1bret_id/step-012932-ckpt.txt

@@ -0,0 +1 @@
+25.193838354546035

out_slim/tiny_LLaMA_1bret_id/step-013923-ckpt.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f148b5338cc3bf69f044eba1ab42ef688f00f6bddd5783cc44746d345818605f
+size 13200815114

out_slim/tiny_LLaMA_1bret_id/step-013927-ckpt.txt

@@ -0,0 +1 @@
+24.447966223118282

out_slim/tiny_LLaMA_1bret_id/step-014922-ckpt.txt

@@ -0,0 +1 @@
+23.815536309986157

out_slim/tiny_LLaMA_1bret_id/step-015912-ckpt.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc344d62e4002e73f695839af3a16ff8410e217c91ee4ee59b91958bbf4139ea
+size 13200815114

out_slim/tiny_LLaMA_1bret_id/step-015917-ckpt.txt

@@ -0,0 +1 @@
+23.431915130712994

out_slim/tiny_LLaMA_1bret_id/step-016912-ckpt.txt

@@ -0,0 +1 @@
+23.674622100643067

out_slim/tiny_LLaMA_1bret_id/step-017901-ckpt.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d8cf2bfe36ebb67f417619218c5fc7b6bd7b4e5a6affb1b9e56a38ab679a560
+size 13200815114

out_slim/tiny_LLaMA_1bret_id/step-017906-ckpt.txt

@@ -0,0 +1 @@
+23.144181770929453

out_slim/tiny_LLaMA_1bret_id/step-018901-ckpt.txt

@@ -0,0 +1 @@
+23.120339733161455

out_slim/tiny_LLaMA_1bret_id/step-019890-ckpt.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d3ba98d95f4111596cecc98a6f56baca76c0bf0e08aec9ec2fd93cdc464d83c
+size 13200815114

out_slim/tiny_LLaMA_1bret_id/step-019896-ckpt.txt

@@ -0,0 +1 @@
+23.02212099004007

out_slim/tiny_LLaMA_1bret_id/wandb/debug.log

@@ -0,0 +1,28 @@
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_setup.py:_flush():76] Current SDK version is 0.15.3
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_setup.py:_flush():76] Configure stats pid to 87
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_setup.py:_flush():76] Loading settings from /home/aiops/zhuty/tinyllama/wandb/settings
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***'}
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'pretrain/tinyllama_mdld.py', 'program': '/home/aiops/zhuty/tinyllama/pretrain/tinyllama_mdld.py'}
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_init.py:_log_setup():507] Logging user logs to out_slim/tiny_LLaMA_1bret_id/wandb/run-20250214_152403-5aszxlzy/logs/debug.log
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_init.py:_log_setup():508] Logging internal logs to out_slim/tiny_LLaMA_1bret_id/wandb/run-20250214_152403-5aszxlzy/logs/debug-internal.log
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_init.py:init():547] calling init triggers
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_init.py:init():554] wandb.init called with sweep_config: {}
+config: {}
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_init.py:init():596] starting backend
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_init.py:init():600] setting up manager
+2025-02-14 15:24:03,521 INFO    MainThread:87 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-02-14 15:24:03,522 INFO    MainThread:87 [wandb_init.py:init():606] backend started and connected
+2025-02-14 15:24:03,525 INFO    MainThread:87 [wandb_init.py:init():700] updated telemetry
+2025-02-14 15:24:03,526 INFO    MainThread:87 [wandb_init.py:init():737] communicating run to backend with 60.0 second timeout
+2025-02-14 15:24:04,462 INFO    MainThread:87 [wandb_run.py:_on_init():2177] communicating current version
+2025-02-14 15:24:04,626 INFO    MainThread:87 [wandb_run.py:_on_init():2186] got version response upgrade_message: "wandb version 0.19.6 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-02-14 15:24:04,626 INFO    MainThread:87 [wandb_init.py:init():787] starting run threads in backend
+2025-02-14 15:24:08,417 INFO    MainThread:87 [wandb_run.py:_console_start():2158] atexit reg
+2025-02-14 15:24:08,417 INFO    MainThread:87 [wandb_run.py:_redirect():2013] redirect: SettingsConsole.WRAP_RAW
+2025-02-14 15:24:08,418 INFO    MainThread:87 [wandb_run.py:_redirect():2078] Wrapping output streams.
+2025-02-14 15:24:08,418 INFO    MainThread:87 [wandb_run.py:_redirect():2103] Redirects installed.
+2025-02-14 15:24:08,418 INFO    MainThread:87 [wandb_init.py:init():829] run started, returning control to user process
+2025-02-15 00:54:21,249 WARNING MsgRouterThr:87 [router.py:message_loop():77] message_loop has been closed

out_slim/tiny_LLaMA_1bret_id/wandb/run-20250214_152403-5aszxlzy/files/conda-environment.yaml

@@ -0,0 +1,333 @@
+name: base
+channels:
+  - pytorch
+  - nvidia
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - asttokens=2.0.5=pyhd3eb1b0_0
+  - backcall=0.2.0=pyhd3eb1b0_0
+  - beautifulsoup4=4.12.2=py310h06a4308_0
+  - blas=1.0=mkl
+  - boltons=23.0.0=py310h06a4308_0
+  - brotlipy=0.7.0=py310h7f8727e_1002
+  - bzip2=1.0.8=h7b6447c_0
+  - c-ares=1.19.0=h5eee18b_0
+  - ca-certificates=2023.08.22=h06a4308_0
+  - certifi=2023.7.22=py310h06a4308_0
+  - cffi=1.15.1=py310h5eee18b_3
+  - chardet=4.0.0=py310h06a4308_1003
+  - charset-normalizer=2.0.4=pyhd3eb1b0_0
+  - click=8.0.4=py310h06a4308_0
+  - cmake=3.26.4=h96355d8_0
+  - conda=23.9.0=py310h06a4308_0
+  - conda-build=3.27.0=py310h06a4308_0
+  - conda-content-trust=0.2.0=py310h06a4308_0
+  - conda-index=0.3.0=py310h06a4308_0
+  - conda-libmamba-solver=23.7.0=py310h06a4308_0
+  - conda-package-handling=2.2.0=py310h06a4308_0
+  - conda-package-streaming=0.9.0=py310h06a4308_0
+  - cryptography=41.0.3=py310hdda0065_0
+  - cuda-cudart=11.8.89=0
+  - cuda-cupti=11.8.87=0
+  - cuda-libraries=11.8.0=0
+  - cuda-nvrtc=11.8.89=0
+  - cuda-nvtx=11.8.86=0
+  - cuda-runtime=11.8.0=0
+  - decorator=5.1.1=pyhd3eb1b0_0
+  - exceptiongroup=1.0.4=py310h06a4308_0
+  - executing=0.8.3=pyhd3eb1b0_0
+  - expat=2.5.0=h6a678d5_0
+  - ffmpeg=4.3=hf484d3e_0
+  - filelock=3.9.0=py310h06a4308_0
+  - fmt=9.1.0=hdb19cb5_0
+  - freetype=2.12.1=h4a9f257_0
+  - giflib=5.2.1=h5eee18b_3
+  - gmp=6.2.1=h295c915_3
+  - gmpy2=2.1.2=py310heeb90bb_0
+  - gnutls=3.6.15=he1e5248_0
+  - icu=58.2=he6710b0_3
+  - idna=3.4=py310h06a4308_0
+  - intel-openmp=2023.1.0=hdb19cb5_46305
+  - ipython=8.15.0=py310h06a4308_0
+  - jedi=0.18.1=py310h06a4308_1
+  - jinja2=3.1.2=py310h06a4308_0
+  - jpeg=9e=h5eee18b_1
+  - jsonpatch=1.32=pyhd3eb1b0_0
+  - jsonpointer=2.1=pyhd3eb1b0_0
+  - krb5=1.20.1=h143b758_1
+  - lame=3.100=h7b6447c_0
+  - lcms2=2.12=h3be6417_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - lerc=3.0=h295c915_0
+  - libarchive=3.6.2=h6ac8c49_2
+  - libcublas=11.11.3.6=0
+  - libcufft=10.9.0.58=0
+  - libcufile=1.7.2.10=0
+  - libcurand=10.3.3.141=0
+  - libcurl=8.1.1=h251f7ec_1
+  - libcusolver=11.4.1.48=0
+  - libcusparse=11.7.5.86=0
+  - libdeflate=1.17=h5eee18b_1
+  - libedit=3.1.20221030=h5eee18b_0
+  - libev=4.33=h7f8727e_1
+  - libffi=3.4.4=h6a678d5_0
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgomp=11.2.0=h1234567_1
+  - libiconv=1.16=h7f8727e_2
+  - libidn2=2.3.4=h5eee18b_0
+  - libjpeg-turbo=2.0.0=h9bf148f_0
+  - liblief=0.12.3=h6a678d5_0
+  - libmamba=1.4.1=h2dafd23_1
+  - libmambapy=1.4.1=py310h2dafd23_1
+  - libnghttp2=1.52.0=h2d74bed_1
+  - libnpp=11.8.0.86=0
+  - libnvjpeg=11.9.0.86=0
+  - libpng=1.6.39=h5eee18b_0
+  - libsolv=0.7.22=he621ea3_0
+  - libssh2=1.10.0=hdbd6064_2
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - libtasn1=4.19.0=h5eee18b_0
+  - libtiff=4.5.1=h6a678d5_0
+  - libunistring=0.9.10=h27cfd23_0
+  - libuuid=1.41.5=h5eee18b_0
+  - libuv=1.44.2=h5eee18b_0
+  - libwebp=1.3.2=h11a3e52_0
+  - libwebp-base=1.3.2=h5eee18b_0
+  - libxml2=2.10.3=hcbfbd50_0
+  - llvm-openmp=14.0.6=h9e868ea_0
+  - lz4-c=1.9.4=h6a678d5_0
+  - markupsafe=2.1.1=py310h7f8727e_0
+  - matplotlib-inline=0.1.6=py310h06a4308_0
+  - mkl=2023.1.0=h213fc3f_46343
+  - mkl-service=2.4.0=py310h5eee18b_1
+  - mkl_fft=1.3.8=py310h5eee18b_0
+  - mkl_random=1.2.4=py310hdb19cb5_0
+  - more-itertools=8.12.0=pyhd3eb1b0_0
+  - mpc=1.1.0=h10f8cd9_1
+  - mpfr=4.0.2=hb69a4c5_1
+  - mpmath=1.3.0=py310h06a4308_0
+  - ncurses=6.4=h6a678d5_0
+  - nettle=3.7.3=hbbd107a_1
+  - networkx=3.1=py310h06a4308_0
+  - numpy=1.26.0=py310h5f9d8c6_0
+  - numpy-base=1.26.0=py310hb5e798b_0
+  - openh264=2.1.1=h4ff587b_0
+  - openssl=3.0.11=h7f8727e_2
+  - packaging=23.1=py310h06a4308_0
+  - parso=0.8.3=pyhd3eb1b0_0
+  - patch=2.7.6=h7b6447c_1001
+  - patchelf=0.17.2=h6a678d5_0
+  - pcre2=10.37=he7ceb23_1
+  - pexpect=4.8.0=pyhd3eb1b0_3
+  - pickleshare=0.7.5=pyhd3eb1b0_1003
+  - pillow=9.4.0=py310h6a678d5_1
+  - pkginfo=1.9.6=py310h06a4308_0
+  - pluggy=1.0.0=py310h06a4308_1
+  - prompt-toolkit=3.0.36=py310h06a4308_0
+  - psutil=5.9.0=py310h5eee18b_0
+  - ptyprocess=0.7.0=pyhd3eb1b0_2
+  - pure_eval=0.2.2=pyhd3eb1b0_0
+  - py-lief=0.12.3=py310h6a678d5_0
+  - pybind11-abi=4=hd3eb1b0_1
+  - pycosat=0.6.4=py310h5eee18b_0
+  - pycparser=2.21=pyhd3eb1b0_0
+  - pygments=2.15.1=py310h06a4308_1
+  - pyopenssl=23.2.0=py310h06a4308_0
+  - pysocks=1.7.1=py310h06a4308_0
+  - python=3.10.13=h955ad1f_0
+  - python-libarchive-c=2.9=pyhd3eb1b0_1
+  - pytorch=2.1.0=py3.10_cuda11.8_cudnn8.7.0_0
+  - pytorch-cuda=11.8=h7e8668a_5
+  - pytorch-mutex=1.0=cuda
+  - pytz=2023.3.post1=py310h06a4308_0
+  - pyyaml=6.0=py310h5eee18b_1
+  - readline=8.2=h5eee18b_0
+  - reproc=14.2.4=h295c915_1
+  - reproc-cpp=14.2.4=h295c915_1
+  - requests=2.31.0=py310h06a4308_0
+  - rhash=1.4.3=hdbd6064_0
+  - ruamel.yaml=0.17.21=py310h5eee18b_0
+  - ruamel.yaml.clib=0.2.6=py310h5eee18b_1
+  - setuptools=68.0.0=py310h06a4308_0
+  - six=1.16.0=pyhd3eb1b0_1
+  - soupsieve=2.5=py310h06a4308_0
+  - sqlite=3.41.2=h5eee18b_0
+  - stack_data=0.2.0=pyhd3eb1b0_0
+  - tbb=2021.8.0=hdb19cb5_0
+  - tk=8.6.12=h1ccaba5_0
+  - tomli=2.0.1=py310h06a4308_0
+  - toolz=0.12.0=py310h06a4308_0
+  - torchaudio=2.1.0=py310_cu118
+  - torchtriton=2.1.0=py310
+  - torchvision=0.16.0=py310_cu118
+  - tqdm=4.65.0=py310h2f386ee_0
+  - traitlets=5.7.1=py310h06a4308_0
+  - truststore=0.8.0=py310h06a4308_0
+  - urllib3=1.26.16=py310h06a4308_0
+  - wcwidth=0.2.5=pyhd3eb1b0_0
+  - wheel=0.41.2=py310h06a4308_0
+  - xz=5.4.2=h5eee18b_0
+  - yaml=0.2.5=h7b6447c_0
+  - yaml-cpp=0.7.0=h295c915_1
+  - zlib=1.2.13=h5eee18b_0
+  - zstandard=0.19.0=py310h5eee18b_0
+  - pip:
+      - accelerate==0.21.0
+      - aiohttp==3.8.6
+      - aiosignal==1.3.1
+      - annotated-types==0.6.0
+      - anyio==3.7.1
+      - appdirs==1.4.4
+      - argon2-cffi==23.1.0
+      - argon2-cffi-bindings==21.2.0
+      - arrow==1.3.0
+      - astunparse==1.6.3
+      - async-lru==2.0.4
+      - async-timeout==4.0.3
+      - attrs==23.1.0
+      - autopep8==2.0.4
+      - babel==2.13.1
+      - backoff==2.2.1
+      - bitsandbytes==0.40.0
+      - bleach==6.1.0
+      - blessed==1.20.0
+      - boto3==1.28.78
+      - botocore==1.31.78
+      - comm==0.1.4
+      - cpplint==1.6.1
+      - croniter==1.4.1
+      - datasets==2.14.6
+      - dateutils==0.6.12
+      - debugpy==1.8.0
+      - deepdiff==6.6.1
+      - defusedxml==0.7.1
+      - dill==0.3.7
+      - dnspython==2.4.2
+      - docker-pycreds==0.4.0
+      - docstring-parser==0.15
+      - dropout-layer-norm==0.1
+      - einops==0.6.1
+      - evaluate==0.4.0
+      - expecttest==0.1.6
+      - fastapi==0.104.1
+      - fastjsonschema==2.18.1
+      - flake8==6.1.0
+      - flash-attn==2.3.3
+      - fqdn==1.5.1
+      - frozenlist==1.4.0
+      - fsspec==2023.9.2
+      - gitdb==4.0.11
+      - gitpython==3.1.40
+      - h11==0.14.0
+      - huggingface-hub==0.18.0
+      - hypothesis==6.87.1
+      - importlib-resources==6.1.0
+      - inquirer==3.1.3
+      - ipykernel==6.26.0
+      - isoduration==20.11.0
+      - isort==5.12.0
+      - itsdangerous==2.1.2
+      - jmespath==1.0.1
+      - joblib==1.3.2
+      - json5==0.9.14
+      - jsonargparse==4.27.0
+      - jsonschema==4.19.2
+      - jsonschema-specifications==2023.7.1
+      - jupyter-client==8.5.0
+      - jupyter-core==5.5.0
+      - jupyter-events==0.8.0
+      - jupyter-lsp==2.2.0
+      - jupyter-server==2.9.1
+      - jupyter-server-terminals==0.4.4
+      - jupyterlab==4.0.8
+      - jupyterlab-pygments==0.2.2
+      - jupyterlab-server==2.25.0
+      - lightning==2.2.0.dev0
+      - lightning-cloud==0.5.44
+      - lightning-utilities==0.9.0
+      - markdown-it-py==3.0.0
+      - mccabe==0.7.0
+      - mdurl==0.1.2
+      - mistune==3.0.2
+      - multidict==6.0.4
+      - multiprocess==0.70.15
+      - nbclient==0.8.0
+      - nbconvert==7.10.0
+      - nbformat==5.9.2
+      - nest-asyncio==1.5.8
+      - ninja==1.11.1.1
+      - notebook-shim==0.2.3
+      - ordered-set==4.1.0
+      - overrides==7.4.0
+      - pandas==2.1.2
+      - pandocfilters==1.5.0
+      - pathtools==0.1.2
+      - peft==0.4.0
+      - pip==23.3.1
+      - platformdirs==3.11.0
+      - prometheus-client==0.18.0
+      - protobuf==4.25.0
+      - pyarrow==14.0.0
+      - pycodestyle==2.11.1
+      - pydantic==2.4.2
+      - pydantic-core==2.10.1
+      - pyflakes==3.1.0
+      - pyjwt==2.8.0
+      - python-dateutil==2.8.2
+      - python-editor==1.0.4
+      - python-etcd==0.4.5
+      - python-json-logger==2.0.7
+      - python-multipart==0.0.6
+      - pytorch-lightning==2.1.0
+      - pyzmq==25.1.1
+      - readchar==4.0.5
+      - referencing==0.30.2
+      - regex==2023.10.3
+      - responses==0.18.0
+      - rfc3339-validator==0.1.4
+      - rfc3986-validator==0.1.1
+      - rich==13.6.0
+      - rotary-emb==0.1
+      - rpds-py==0.12.0
+      - s3transfer==0.7.0
+      - safetensors==0.4.0
+      - scikit-learn==1.2.2
+      - scipy==1.11.3
+      - send2trash==1.8.2
+      - sentencepiece==0.1.99
+      - sentry-sdk==1.34.0
+      - setproctitle==1.3.3
+      - smmap==5.0.1
+      - sniffio==1.3.0
+      - sortedcontainers==2.4.0
+      - starlette==0.27.0
+      - starsessions==1.3.0
+      - sympy==1.12
+      - terminado==0.17.1
+      - threadpoolctl==3.2.0
+      - tinycss2==1.2.1
+      - tokenizers==0.13.3
+      - torchelastic==0.2.2
+      - torchmetrics==1.2.0
+      - tornado==6.3.3
+      - transformers==4.31.0
+      - types-dataclasses==0.6.6
+      - types-python-dateutil==2.8.19.14
+      - typeshed-client==2.4.0
+      - typing-extensions==4.8.0
+      - tzdata==2023.3
+      - uri-template==1.3.0
+      - uvicorn==0.24.0
+      - wandb==0.15.3
+      - webcolors==1.13
+      - webencodings==0.5.1
+      - websocket-client==1.6.4
+      - websockets==11.0.3
+      - xentropy-cuda-lib==0.1
+      - xformers==0.0.22.post7+cu118
+      - xxhash==3.4.1
+      - yarl==1.9.2
+      - zstd==1.5.5.1
+prefix: /opt/conda

out_slim/tiny_LLaMA_1bret_id/wandb/run-20250214_152403-5aszxlzy/files/config.yaml

@@ -0,0 +1,110 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.13
+    cli_version: 0.15.3
+    framework: huggingface
+    huggingface_version: 4.31.0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1739546643.523855
+    t:
+      1:
+      - 1
+      - 9
+      - 11
+      - 41
+      - 49
+      - 55
+      - 71
+      2:
+      - 1
+      - 9
+      - 11
+      - 41
+      - 49
+      - 55
+      - 71
+      3:
+      - 7
+      - 13
+      - 23
+      4: 3.10.13
+      5: 0.15.3
+      6: 4.31.0
+      8:
+      - 5
+    m:
+    - 1: trainer/global_step
+      6:
+      - 3
+    - 1: time/train
+      5: 1
+      6:
+      - 1
+    - 1: time/val
+      5: 1
+      6:
+      - 1
+    - 1: time/total
+      5: 1
+      6:
+      - 1
+    - 1: samples
+      5: 1
+      6:
+      - 1
+    - 1: throughput/batches_per_sec
+      5: 1
+      6:
+      - 1
+    - 1: throughput/samples_per_sec
+      5: 1
+      6:
+      - 1
+    - 1: throughput/device/batches_per_sec
+      5: 1
+      6:
+      - 1
+    - 1: throughput/device/samples_per_sec
+      5: 1
+      6:
+      - 1
+    - 1: throughput/tokens_per_sec
+      5: 1
+      6:
+      - 1
+    - 1: throughput/device/tokens_per_sec
+      5: 1
+      6:
+      - 1
+    - 1: total_tokens
+      5: 1
+      6:
+      - 1
+    - 1: metric/train_loss
+      5: 1
+      6:
+      - 1
+    - 1: metric/train_ppl
+      5: 1
+      6:
+      - 1
+    - 1: throughput/flops_per_sec
+      5: 1
+      6:
+      - 1
+    - 1: throughput/device/flops_per_sec
+      5: 1
+      6:
+      - 1
+    - 1: throughput/device/mfu
+      5: 1
+      6:
+      - 1
+    - 1: metric/val_loss
+      5: 1
+      6:
+      - 1

out_slim/tiny_LLaMA_1bret_id/wandb/run-20250214_152403-5aszxlzy/files/requirements.txt

@@ -0,0 +1,234 @@
+accelerate==0.21.0
+aiohttp==3.8.6
+aiosignal==1.3.1
+annotated-types==0.6.0
+anyio==3.7.1
+appdirs==1.4.4
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+arrow==1.3.0
+asttokens==2.0.5
+astunparse==1.6.3
+async-lru==2.0.4
+async-timeout==4.0.3
+attrs==23.1.0
+autopep8==2.0.4
+babel==2.13.1
+backcall==0.2.0
+backoff==2.2.1
+beautifulsoup4==4.12.2
+bitsandbytes==0.40.0
+bleach==6.1.0
+blessed==1.20.0
+boltons==23.0.0
+boto3==1.28.78
+botocore==1.31.78
+brotlipy==0.7.0
+certifi==2023.7.22
+cffi==1.15.1
+chardet==4.0.0
+charset-normalizer==2.0.4
+click==8.0.4
+comm==0.1.4
+conda-build==3.27.0
+conda-content-trust==0.2.0
+conda-index==0.3.0
+conda-libmamba-solver==23.7.0
+conda-package-handling==2.2.0
+conda-package-streaming==0.9.0
+conda==23.9.0
+cpplint==1.6.1
+croniter==1.4.1
+cryptography==41.0.3
+datasets==2.14.6
+dateutils==0.6.12
+debugpy==1.8.0
+decorator==5.1.1
+deepdiff==6.6.1
+defusedxml==0.7.1
+dill==0.3.7
+dnspython==2.4.2
+docker-pycreds==0.4.0
+docstring-parser==0.15
+dropout-layer-norm==0.1
+einops==0.6.1
+evaluate==0.4.0
+exceptiongroup==1.0.4
+executing==0.8.3
+expecttest==0.1.6
+fastapi==0.104.1
+fastjsonschema==2.18.1
+filelock==3.9.0
+flake8==6.1.0
+flash-attn==2.3.3
+fqdn==1.5.1
+frozenlist==1.4.0
+fsspec==2023.9.2
+gitdb==4.0.11
+gitpython==3.1.40
+gmpy2==2.1.2
+h11==0.14.0
+huggingface-hub==0.18.0
+hypothesis==6.87.1
+idna==3.4
+importlib-resources==6.1.0
+inquirer==3.1.3
+ipykernel==6.26.0
+ipython==8.15.0
+isoduration==20.11.0
+isort==5.12.0
+itsdangerous==2.1.2
+jedi==0.18.1
+jinja2==3.1.2
+jmespath==1.0.1
+joblib==1.3.2
+json5==0.9.14
+jsonargparse==4.27.0
+jsonpatch==1.32
+jsonpointer==2.1
+jsonschema-specifications==2023.7.1
+jsonschema==4.19.2
+jupyter-client==8.5.0
+jupyter-core==5.5.0
+jupyter-events==0.8.0
+jupyter-lsp==2.2.0
+jupyter-server-terminals==0.4.4
+jupyter-server==2.9.1
+jupyterlab-pygments==0.2.2
+jupyterlab-server==2.25.0
+jupyterlab==4.0.8
+libarchive-c==2.9
+libmambapy==1.4.1
+lightning-cloud==0.5.44
+lightning-utilities==0.9.0
+lightning==2.2.0.dev0
+markdown-it-py==3.0.0
+markupsafe==2.1.1
+matplotlib-inline==0.1.6
+mccabe==0.7.0
+mdurl==0.1.2
+mistune==3.0.2
+mkl-fft==1.3.8
+mkl-random==1.2.4
+mkl-service==2.4.0
+more-itertools==8.12.0
+mpmath==1.3.0
+multidict==6.0.4
+multiprocess==0.70.15
+nbclient==0.8.0
+nbconvert==7.10.0
+nbformat==5.9.2
+nest-asyncio==1.5.8
+networkx==3.1
+ninja==1.11.1.1
+notebook-shim==0.2.3
+numpy==1.26.0
+ordered-set==4.1.0
+overrides==7.4.0
+packaging==23.1
+pandas==2.1.2
+pandocfilters==1.5.0
+parso==0.8.3
+pathtools==0.1.2
+peft==0.4.0
+pexpect==4.8.0
+pickleshare==0.7.5
+pillow==9.4.0
+pip==23.3.1
+pkginfo==1.9.6
+platformdirs==3.11.0
+pluggy==1.0.0
+prometheus-client==0.18.0
+prompt-toolkit==3.0.36
+protobuf==4.25.0
+psutil==5.9.0
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow==14.0.0
+pycodestyle==2.11.1
+pycosat==0.6.4
+pycparser==2.21
+pydantic-core==2.10.1
+pydantic==2.4.2
+pyflakes==3.1.0
+pygments==2.15.1
+pyjwt==2.8.0
+pyopenssl==23.2.0
+pysocks==1.7.1
+python-dateutil==2.8.2
+python-editor==1.0.4
+python-etcd==0.4.5
+python-json-logger==2.0.7
+python-multipart==0.0.6
+pytorch-lightning==2.1.0
+pytz==2023.3.post1
+pyyaml==6.0
+pyzmq==25.1.1
+readchar==4.0.5
+referencing==0.30.2
+regex==2023.10.3
+requests==2.31.0
+responses==0.18.0
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==13.6.0
+rotary-emb==0.1
+rpds-py==0.12.0
+ruamel.yaml.clib==0.2.6
+ruamel.yaml==0.17.21
+s3transfer==0.7.0
+safetensors==0.4.0
+scikit-learn==1.2.2
+scipy==1.11.3
+send2trash==1.8.2
+sentencepiece==0.1.99
+sentry-sdk==1.34.0
+setproctitle==1.3.3
+setuptools==68.0.0
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.0
+sortedcontainers==2.4.0
+soupsieve==2.5
+stack-data==0.2.0
+starlette==0.27.0
+starsessions==1.3.0
+sympy==1.11.1
+terminado==0.17.1
+threadpoolctl==3.2.0
+tinycss2==1.2.1
+tokenizers==0.13.3
+tomli==2.0.1
+toolz==0.12.0
+torch==2.1.0
+torchaudio==2.1.0
+torchelastic==0.2.2
+torchmetrics==1.2.0
+torchvision==0.16.0
+tornado==6.3.3
+tqdm==4.65.0
+traitlets==5.7.1
+transformers==4.31.0
+triton==2.1.0
+truststore==0.8.0
+types-dataclasses==0.6.6
+types-python-dateutil==2.8.19.14
+typeshed-client==2.4.0
+typing-extensions==4.8.0
+tzdata==2023.3
+uri-template==1.3.0
+urllib3==1.26.16
+uvicorn==0.24.0
+wandb==0.15.3
+wcwidth==0.2.5
+webcolors==1.13
+webencodings==0.5.1
+websocket-client==1.6.4
+websockets==11.0.3
+wheel==0.41.2
+xentropy-cuda-lib==0.1
+xformers==0.0.22.post7+cu118
+xxhash==3.4.1
+yarl==1.9.2
+zstandard==0.19.0
+zstd==1.5.5.1

out_slim/tiny_LLaMA_1bret_id/wandb/run-20250214_152403-5aszxlzy/files/wandb-metadata.json

@@ -0,0 +1,1048 @@
+{
+    "os": "Linux-5.15.0-18-shopee-generic-x86_64-with-glibc2.31",
+    "python": "3.10.13",
+    "heartbeatAt": "2025-02-14T15:24:04.720319",
+    "startedAt": "2025-02-14T15:24:03.453895",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--num_of_device",
+        "8",
+        "--model_name",
+        "tiny_LLaMA_1b",
+        "--exp_name",
+        "ret_id",
+        "--max_step",
+        "19896",
+        "--warmup_steps",
+        "1989",
+        "--save_step_interval",
+        "1989",
+        "--micro_batch_size_para",
+        "8",
+        "--resume",
+        "True",
+        "--train_data_dir",
+        "/sailor2/tongyao/MADLAD-400-noisy/data/idtop20_bins",
+        "--val_data_dir",
+        "/home/aiops/zhuty/ret_pretraining_data/sample_processed/tr_wiki_2k"
+    ],
+    "state": "running",
+    "program": "/home/aiops/zhuty/tinyllama/pretrain/tinyllama_mdld.py",
+    "codePath": "pretrain/tinyllama_mdld.py",
+    "host": "tmkv-1-10-237-0-29",
+    "username": "root",
+    "executable": "/opt/conda/bin/python",
+    "cpu_count": 96,
+    "cpu_count_logical": 192,
+    "cpu_freq": {
+        "current": 2.074765625000005,
+        "min": 1500.0,
+        "max": 2300.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 3.443,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.9,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 3.417,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.9,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 3.417,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 3.417,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 3.417,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 3.417,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 3.417,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 3.417,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 3.417,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.9,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 1.5,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 3.443,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 3.443,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 3.417,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 3.416,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 3.414,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 3.417,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        },
+        {
+            "current": 2.3,
+            "min": 1500.0,
+            "max": 2300.0
+        }
+    ],
+    "disk": {
+        "total": 438.80956268310547,
+        "used": 299.2193603515625
+    },
+    "gpu": "NVIDIA A100-SXM4-40GB",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        },
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        },
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        },
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        },
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        },
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        },
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        },
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        }
+    ],
+    "memory": {
+        "total": 1007.0852203369141
+    }
+}

out_slim/tiny_LLaMA_1bret_id/wandb/run-20250214_152403-5aszxlzy/files/wandb-summary.json

@@ -0,0 +1 @@
+{"time/train": 33901.23735222593, "time/val": 5863.853048056364, "time/total": 39765.09040028229, "samples": 318248, "trainer/global_step": 19896, "_timestamp": 1739580852.5422957, "_runtime": 34209.01844072342, "_step": 2009, "throughput/batches_per_sec": 1.4046060805526, "throughput/samples_per_sec": 11.2368486444208, "throughput/device/batches_per_sec": 0.04389394001726875, "throughput/device/samples_per_sec": 0.35115152013815, "throughput/tokens_per_sec": 23013.066023773797, "throughput/device/tokens_per_sec": 719.1583132429312, "total_tokens": 20862468096, "metric/train_loss": 1.1789775103330613, "metric/train_ppl": 3.2510483395150933, "throughput/flops_per_sec": 194363386287368.6, "throughput/device/flops_per_sec": 6073855821480.269, "throughput/device/mfu": 0.019467486607308553, "metric/val_loss": 3.1413323879241943, "_wandb": {"runtime": 34208}}

out_slim/tiny_LLaMA_1bret_id/wandb/run-20250214_152403-5aszxlzy/logs/debug.log

@@ -0,0 +1,28 @@
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_setup.py:_flush():76] Current SDK version is 0.15.3
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_setup.py:_flush():76] Configure stats pid to 87
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_setup.py:_flush():76] Loading settings from /home/aiops/zhuty/tinyllama/wandb/settings
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***'}
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'pretrain/tinyllama_mdld.py', 'program': '/home/aiops/zhuty/tinyllama/pretrain/tinyllama_mdld.py'}
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_init.py:_log_setup():507] Logging user logs to out_slim/tiny_LLaMA_1bret_id/wandb/run-20250214_152403-5aszxlzy/logs/debug.log
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_init.py:_log_setup():508] Logging internal logs to out_slim/tiny_LLaMA_1bret_id/wandb/run-20250214_152403-5aszxlzy/logs/debug-internal.log
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_init.py:init():547] calling init triggers
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_init.py:init():554] wandb.init called with sweep_config: {}
+config: {}
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_init.py:init():596] starting backend
+2025-02-14 15:24:03,520 INFO    MainThread:87 [wandb_init.py:init():600] setting up manager
+2025-02-14 15:24:03,521 INFO    MainThread:87 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-02-14 15:24:03,522 INFO    MainThread:87 [wandb_init.py:init():606] backend started and connected
+2025-02-14 15:24:03,525 INFO    MainThread:87 [wandb_init.py:init():700] updated telemetry
+2025-02-14 15:24:03,526 INFO    MainThread:87 [wandb_init.py:init():737] communicating run to backend with 60.0 second timeout
+2025-02-14 15:24:04,462 INFO    MainThread:87 [wandb_run.py:_on_init():2177] communicating current version
+2025-02-14 15:24:04,626 INFO    MainThread:87 [wandb_run.py:_on_init():2186] got version response upgrade_message: "wandb version 0.19.6 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-02-14 15:24:04,626 INFO    MainThread:87 [wandb_init.py:init():787] starting run threads in backend
+2025-02-14 15:24:08,417 INFO    MainThread:87 [wandb_run.py:_console_start():2158] atexit reg
+2025-02-14 15:24:08,417 INFO    MainThread:87 [wandb_run.py:_redirect():2013] redirect: SettingsConsole.WRAP_RAW
+2025-02-14 15:24:08,418 INFO    MainThread:87 [wandb_run.py:_redirect():2078] Wrapping output streams.
+2025-02-14 15:24:08,418 INFO    MainThread:87 [wandb_run.py:_redirect():2103] Redirects installed.
+2025-02-14 15:24:08,418 INFO    MainThread:87 [wandb_init.py:init():829] run started, returning control to user process
+2025-02-15 00:54:21,249 WARNING MsgRouterThr:87 [router.py:message_loop():77] message_loop has been closed

out_slim/tiny_LLaMA_1bret_id/wandb/run-20250214_152403-5aszxlzy/run-5aszxlzy.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59c306e14ee993bcf59c661af2c21cc074e958cdcbbe7fb26006219b9c7cf601
+size 13031098