granularity results ultraep/tierbtrace @ 2026-08-26T20:23:21Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0
Browse files- .gitattributes +1 -0
- ultraep/tierbtrace/tierbtrace_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/nsys/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375.nsys-rep +3 -0
- ultraep/tierbtrace/tierbtrace_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/nsys/train.log +68 -0
- ultraep/tierbtrace/tierbtrace_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/nsys/verify_trace_ranks.log +5 -0
- ultraep/tierbtrace/tierbtrace_gbs8/trace_results.tsv +1 -0
.gitattributes
CHANGED
|
@@ -52,3 +52,4 @@ echo/tierbtrace/tierbtrace_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p08
|
|
| 52 |
ultraep/tierbtrace/tierbtrace_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p083984375/nsys/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p083984375.nsys-rep filter=lfs diff=lfs merge=lfs -text
|
| 53 |
ultraep/tierbtrace/tierbtrace_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/nsys/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125.nsys-rep filter=lfs diff=lfs merge=lfs -text
|
| 54 |
ultraep/tierbtrace/tierbtrace_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/nsys/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333.nsys-rep filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 52 |
ultraep/tierbtrace/tierbtrace_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p083984375/nsys/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p083984375.nsys-rep filter=lfs diff=lfs merge=lfs -text
|
| 53 |
ultraep/tierbtrace/tierbtrace_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/nsys/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125.nsys-rep filter=lfs diff=lfs merge=lfs -text
|
| 54 |
ultraep/tierbtrace/tierbtrace_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/nsys/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333.nsys-rep filter=lfs diff=lfs merge=lfs -text
|
| 55 |
+
ultraep/tierbtrace/tierbtrace_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/nsys/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375.nsys-rep filter=lfs diff=lfs merge=lfs -text
|
ultraep/tierbtrace/tierbtrace_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/nsys/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375.nsys-rep
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:241d8ed293d5c4639cd41be94792c12d225491dbe16508045d6e899768cd34aa
|
| 3 |
+
size 5790608
|
ultraep/tierbtrace/tierbtrace_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/nsys/train.log
CHANGED
|
@@ -1149,3 +1149,71 @@ CUDA graph capture done!!!
|
|
| 1149 |
[2026-08-26 20:13:59] iteration 6/ 10 | consumed samples: 48 | elapsed time per iteration (ms): 192.1 | throughput per GPU (TFLOP/s/GPU): 642.8 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.050245E+01 | loss scale: 1.0 | grad norm: 8.738 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 |
|
| 1150 |
[2026-08-26 20:14:02] iteration 7/ 10 | consumed samples: 56 | elapsed time per iteration (ms): 2933.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.451855E+00 | loss scale: 1.0 | grad norm: 5.610 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 |
|
| 1151 |
[2026-08-26 20:14:02] iteration 8/ 10 | consumed samples: 64 | elapsed time per iteration (ms): 202.4 | throughput per GPU (TFLOP/s/GPU): 610.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.983458E+00 | loss scale: 1.0 | grad norm: 4.580 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1149 |
[2026-08-26 20:13:59] iteration 6/ 10 | consumed samples: 48 | elapsed time per iteration (ms): 192.1 | throughput per GPU (TFLOP/s/GPU): 642.8 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.050245E+01 | loss scale: 1.0 | grad norm: 8.738 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 |
|
| 1150 |
[2026-08-26 20:14:02] iteration 7/ 10 | consumed samples: 56 | elapsed time per iteration (ms): 2933.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.451855E+00 | loss scale: 1.0 | grad norm: 5.610 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 |
|
| 1151 |
[2026-08-26 20:14:02] iteration 8/ 10 | consumed samples: 64 | elapsed time per iteration (ms): 202.4 | throughput per GPU (TFLOP/s/GPU): 610.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.983458E+00 | loss scale: 1.0 | grad norm: 4.580 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 |
|
| 1152 |
+
Capture range started in the application.
|
| 1153 |
+
Capture range ended in the application.
|
| 1154 |
+
Generating '/tmp/nsys-report-f9bd.qdstrm'
|
| 1155 |
+
|
| 1156 |
+
|
| 1157 |
+
[2026-08-26 20:14:41] iteration 9/ 10 | consumed samples: 72 | elapsed time per iteration (ms): 39305.3 | throughput per GPU (TFLOP/s/GPU): 3.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.769887E+00 | loss scale: 1.0 | grad norm: 4.334 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 |
|
| 1158 |
+
[2026-08-26 20:14:42] iteration 10/ 10 | consumed samples: 80 | elapsed time per iteration (ms): 195.9 | throughput per GPU (TFLOP/s/GPU): 630.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.710808E+00 | loss scale: 1.0 | grad norm: 4.469 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 |
|
| 1159 |
+
[after training is done] datetime: 2026-08-26 20:14:42
|
| 1160 |
+
[rank0]:[W826 20:14:43.388230076 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
|
| 1161 |
+
[rank1]:[W826 20:14:44.620278979 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
|
| 1162 |
+
[rank2]:[W826 20:14:44.628488724 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
|
| 1163 |
+
[rank3]:[W826 20:14:44.633798372 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
|
| 1164 |
+
Collection stop is not allowed in this state.
|
| 1165 |
+
[rank7]:[W826 20:14:45.058367801 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
|
| 1166 |
+
[rank5]:[W826 20:14:45.059115495 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
|
| 1167 |
+
[rank6]:[W826 20:14:45.092607803 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
|
| 1168 |
+
[rank4]:[W826 20:14:45.513426251 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
|
| 1169 |
+
[rank0]:[W826 20:14:48.089209129 observer.cpp:116] Warning: Leaked callback handle: 1 (function operator())
|
| 1170 |
+
pure virtual method called
|
| 1171 |
+
terminate called without an active exception
|
| 1172 |
+
[rank2]:[W826 20:14:48.109237585 observer.cpp:116] Warning: Leaked callback handle: 1 (function operator())
|
| 1173 |
+
pure virtual method called
|
| 1174 |
+
terminate called without an active exception
|
| 1175 |
+
[rank3]:[W826 20:14:48.177611155 observer.cpp:116] Warning: Leaked callback handle: 1 (function operator())
|
| 1176 |
+
pure virtual method called
|
| 1177 |
+
terminate called without an active exception
|
| 1178 |
+
W0826 20:14:48.625000 755703 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 755813 closing signal SIGTERM
|
| 1179 |
+
W0826 20:14:48.626000 755703 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 755814 closing signal SIGTERM
|
| 1180 |
+
W0826 20:14:48.627000 755703 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 755815 closing signal SIGTERM
|
| 1181 |
+
W0826 20:14:48.628000 755703 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 755816 closing signal SIGTERM
|
| 1182 |
+
W0826 20:14:48.628000 755703 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 755817 closing signal SIGTERM
|
| 1183 |
+
W0826 20:14:48.630000 755703 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 755818 closing signal SIGTERM
|
| 1184 |
+
W0826 20:14:48.632000 755703 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 755819 closing signal SIGTERM
|
| 1185 |
+
E0826 20:14:50.364000 755703 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: -6) local_rank: 0 (pid: 755812) of binary: /usr/bin/python
|
| 1186 |
+
Traceback (most recent call last):
|
| 1187 |
+
File "/usr/local/bin/torchrun", line 8, in <module>
|
| 1188 |
+
sys.exit(main())
|
| 1189 |
+
^^^^^^
|
| 1190 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
|
| 1191 |
+
return f(*args, **kwargs)
|
| 1192 |
+
^^^^^^^^^^^^^^^^^^
|
| 1193 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main
|
| 1194 |
+
run(args)
|
| 1195 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run
|
| 1196 |
+
elastic_launch(
|
| 1197 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__
|
| 1198 |
+
return launch_agent(self._config, self._entrypoint, list(args))
|
| 1199 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 1200 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent
|
| 1201 |
+
raise ChildFailedError(
|
| 1202 |
+
torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
|
| 1203 |
+
============================================================
|
| 1204 |
+
pretrain_gpt.py FAILED
|
| 1205 |
+
------------------------------------------------------------
|
| 1206 |
+
Failures:
|
| 1207 |
+
<NO_OTHER_FAILURES>
|
| 1208 |
+
------------------------------------------------------------
|
| 1209 |
+
Root Cause (first observed failure):
|
| 1210 |
+
[0]:
|
| 1211 |
+
time : 2026-08-26_20:14:48
|
| 1212 |
+
host : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0
|
| 1213 |
+
rank : 0 (local_rank: 0)
|
| 1214 |
+
exitcode : -6 (pid: 755812)
|
| 1215 |
+
error_file: <N/A>
|
| 1216 |
+
traceback : Signal 6 (SIGABRT) received by PID 755812
|
| 1217 |
+
============================================================
|
| 1218 |
+
Generated:
|
| 1219 |
+
/workspace/tierb_study_20260826_144644/ultraep/tierbtrace/tierbtrace_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/nsys/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375.nsys-rep
|
ultraep/tierbtrace/tierbtrace_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/nsys/verify_trace_ranks.log
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
report : /workspace/tierb_study_20260826_144644/ultraep/tierbtrace/tierbtrace_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/nsys/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375.nsys-rep
|
| 2 |
+
kernels : 54110
|
| 3 |
+
devices : 8 of 8 -> [0, 1, 2, 3, 4, 5, 6, 7]
|
| 4 |
+
processes: 1
|
| 5 |
+
PASS: all ranks present
|
ultraep/tierbtrace/tierbtrace_gbs8/trace_results.tsv
CHANGED
|
@@ -2,3 +2,4 @@ model backend capacity status report
|
|
| 2 |
QWEN3_235B ultraep 1.083984375 OK /workspace/tierb_study_20260826_144644/ultraep/tierbtrace/tierbtrace_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p083984375/nsys/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p083984375.nsys-rep
|
| 3 |
QWEN3_30B ultraep 1.11328125 OK /workspace/tierb_study_20260826_144644/ultraep/tierbtrace/tierbtrace_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/nsys/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125.nsys-rep
|
| 4 |
DSV2_LITE_L24 ultraep 1.0755208333333333 OK /workspace/tierb_study_20260826_144644/ultraep/tierbtrace/tierbtrace_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/nsys/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333.nsys-rep
|
|
|
|
|
|
| 2 |
QWEN3_235B ultraep 1.083984375 OK /workspace/tierb_study_20260826_144644/ultraep/tierbtrace/tierbtrace_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p083984375/nsys/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p083984375.nsys-rep
|
| 3 |
QWEN3_30B ultraep 1.11328125 OK /workspace/tierb_study_20260826_144644/ultraep/tierbtrace/tierbtrace_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125/nsys/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p11328125.nsys-rep
|
| 4 |
DSV2_LITE_L24 ultraep 1.0755208333333333 OK /workspace/tierb_study_20260826_144644/ultraep/tierbtrace/tierbtrace_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333/nsys/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0755208333333333.nsys-rep
|
| 5 |
+
QWEN2_57B ultraep 1.08984375 OK /workspace/tierb_study_20260826_144644/ultraep/tierbtrace/tierbtrace_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375/nsys/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p08984375.nsys-rep
|