granularity results echo/tierbtrace @ 2026-08-26T19:52:54Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0
Browse files- .gitattributes +1 -0
- echo/tierbtrace/tierbtrace_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0755208333333333/nsys/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0755208333333333.nsys-rep +3 -0
- echo/tierbtrace/tierbtrace_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0755208333333333/nsys/run.env +61 -0
- echo/tierbtrace/tierbtrace_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0755208333333333/nsys/train.log +0 -0
- echo/tierbtrace/tierbtrace_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125/nsys/train.log +44 -0
- echo/tierbtrace/tierbtrace_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125/nsys/verify_trace_ranks.log +5 -0
- echo/tierbtrace/tierbtrace_gbs8/trace_results.tsv +1 -0
.gitattributes
CHANGED
|
@@ -47,3 +47,4 @@ echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00
|
|
| 47 |
echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log filter=lfs diff=lfs merge=lfs -text
|
| 48 |
echo/tierbtrace/tierbtrace_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p083984375/nsys/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p083984375.nsys-rep filter=lfs diff=lfs merge=lfs -text
|
| 49 |
echo/tierbtrace/tierbtrace_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125/nsys/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125.nsys-rep filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 47 |
echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log filter=lfs diff=lfs merge=lfs -text
|
| 48 |
echo/tierbtrace/tierbtrace_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p083984375/nsys/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p083984375.nsys-rep filter=lfs diff=lfs merge=lfs -text
|
| 49 |
echo/tierbtrace/tierbtrace_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125/nsys/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125.nsys-rep filter=lfs diff=lfs merge=lfs -text
|
| 50 |
+
echo/tierbtrace/tierbtrace_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0755208333333333/nsys/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0755208333333333.nsys-rep filter=lfs diff=lfs merge=lfs -text
|
echo/tierbtrace/tierbtrace_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0755208333333333/nsys/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0755208333333333.nsys-rep
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bb0818748e309a82e483546035725dfb2b5a27d5507aa596a053474c7d87ac49
|
| 3 |
+
size 17103761
|
echo/tierbtrace/tierbtrace_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0755208333333333/nsys/run.env
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=echo
|
| 2 |
+
config=DSV2_LITE_L24
|
| 3 |
+
family=tierB
|
| 4 |
+
repo=/workspace/Megatron-MoE-Echo
|
| 5 |
+
branch=bracing-regions-through-kernels
|
| 6 |
+
commit=9a236793
|
| 7 |
+
num_layers=24
|
| 8 |
+
hidden=2048
|
| 9 |
+
ffn_hidden=10944
|
| 10 |
+
moe_ffn_hidden=1408
|
| 11 |
+
num_experts=64
|
| 12 |
+
topk=6
|
| 13 |
+
experts_per_rank=8
|
| 14 |
+
expert_params=8650752
|
| 15 |
+
heads=16
|
| 16 |
+
kv_groups=0
|
| 17 |
+
kv_channels=128
|
| 18 |
+
qk_layernorm=1
|
| 19 |
+
shared_ffn=2816
|
| 20 |
+
shared_expert_overlap=0
|
| 21 |
+
mla=1
|
| 22 |
+
dense_first_layers=1
|
| 23 |
+
moe_layers=23
|
| 24 |
+
rotary_base=10000
|
| 25 |
+
rotary_scaling=40
|
| 26 |
+
qkv_bias=0
|
| 27 |
+
router_balancing=seq_aux_loss
|
| 28 |
+
aux_loss_coeff=1e-3
|
| 29 |
+
router_pre_softmax=1
|
| 30 |
+
router_fusion=0
|
| 31 |
+
router_dtype=fp32
|
| 32 |
+
device_initiated_grouped_gemm=1
|
| 33 |
+
norm_epsilon=1e-6
|
| 34 |
+
seq_length=8192
|
| 35 |
+
micro_batch_size=1
|
| 36 |
+
global_batch_size=8
|
| 37 |
+
num_microbatches=1
|
| 38 |
+
assignments_per_rank=49152
|
| 39 |
+
mean_tokens_per_expert=6144
|
| 40 |
+
slot_policy=matched
|
| 41 |
+
slots_per_rank=8
|
| 42 |
+
capacity_mode=static
|
| 43 |
+
capacity=1.0755208333333333
|
| 44 |
+
cuda_graph=1
|
| 45 |
+
probe_mode=0
|
| 46 |
+
async_grad_reduce=0
|
| 47 |
+
branch_role=timing
|
| 48 |
+
memory_dump_mode=none
|
| 49 |
+
memory_snapshot_path=
|
| 50 |
+
cuda_graph_warmup_steps=3
|
| 51 |
+
recompute=none
|
| 52 |
+
force_balance=0
|
| 53 |
+
train_iters=10
|
| 54 |
+
ep_size=8
|
| 55 |
+
echo_dispatch_overlap=1
|
| 56 |
+
echo_dispatch_recompute=0
|
| 57 |
+
ultraep_quota_min_tokens=
|
| 58 |
+
ultraep_weight_sync_join=
|
| 59 |
+
profile_enable=1
|
| 60 |
+
kernel_trace=1
|
| 61 |
+
fp8_recipe=mxfp8
|
echo/tierbtrace/tierbtrace_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0755208333333333/nsys/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
echo/tierbtrace/tierbtrace_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125/nsys/train.log
CHANGED
|
@@ -2803,3 +2803,47 @@ Capture range ended in the application.
|
|
| 2803 |
Generating '/tmp/nsys-report-f6e5.qdstrm'
|
| 2804 |
|
| 2805 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2803 |
Generating '/tmp/nsys-report-f6e5.qdstrm'
|
| 2804 |
|
| 2805 |
|
| 2806 |
+
Collection stop is not allowed in this state.
|
| 2807 |
+
W0826 19:46:13.770000 700305 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers
|
| 2808 |
+
W0826 19:46:13.771000 700305 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 700421 closing signal SIGTERM
|
| 2809 |
+
Generated:
|
| 2810 |
+
/workspace/tierb_study_20260826_144644/echo/tierbtrace/tierbtrace_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125/nsys/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125.nsys-rep
|
| 2811 |
+
W0826 19:46:13.771000 700305 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 700422 closing signal SIGTERM
|
| 2812 |
+
W0826 19:46:13.789000 700305 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 700423 closing signal SIGTERM
|
| 2813 |
+
W0826 19:46:13.812000 700305 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 700424 closing signal SIGTERM
|
| 2814 |
+
W0826 19:46:13.832000 700305 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 700425 closing signal SIGTERM
|
| 2815 |
+
W0826 19:46:13.841000 700305 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 700426 closing signal SIGTERM
|
| 2816 |
+
W0826 19:46:13.851000 700305 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 700427 closing signal SIGTERM
|
| 2817 |
+
W0826 19:46:13.866000 700305 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 700428 closing signal SIGTERM
|
| 2818 |
+
Traceback (most recent call last):
|
| 2819 |
+
File "/usr/local/bin/torchrun", line 8, in <module>
|
| 2820 |
+
sys.exit(main())
|
| 2821 |
+
^^^^^^
|
| 2822 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
|
| 2823 |
+
return f(*args, **kwargs)
|
| 2824 |
+
^^^^^^^^^^^^^^^^^^
|
| 2825 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main
|
| 2826 |
+
run(args)
|
| 2827 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run
|
| 2828 |
+
elastic_launch(
|
| 2829 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__
|
| 2830 |
+
return launch_agent(self._config, self._entrypoint, list(args))
|
| 2831 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 2832 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent
|
| 2833 |
+
result = agent.run()
|
| 2834 |
+
^^^^^^^^^^^
|
| 2835 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
| 2836 |
+
result = f(*args, **kwargs)
|
| 2837 |
+
^^^^^^^^^^^^^^^^^^
|
| 2838 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run
|
| 2839 |
+
result = self._invoke_run(role)
|
| 2840 |
+
^^^^^^^^^^^^^^^^^^^^^^
|
| 2841 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run
|
| 2842 |
+
time.sleep(monitor_interval)
|
| 2843 |
+
File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler
|
| 2844 |
+
raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
|
| 2845 |
+
torch.distributed.elastic.multiprocessing.api.SignalException: Process 700305 got signal: 15
|
| 2846 |
+
The target application terminated. One or more process it created re-parented.
|
| 2847 |
+
Waiting for termination of re-parented processes.
|
| 2848 |
+
Use the `--wait` option to modify this behavior.
|
| 2849 |
+
There are no active sessions.
|
echo/tierbtrace/tierbtrace_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125/nsys/verify_trace_ranks.log
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
report : /workspace/tierb_study_20260826_144644/echo/tierbtrace/tierbtrace_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125/nsys/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125.nsys-rep
|
| 2 |
+
kernels : 398480
|
| 3 |
+
devices : 8 of 8 -> [0, 1, 2, 3, 4, 5, 6, 7]
|
| 4 |
+
processes: 1
|
| 5 |
+
PASS: all ranks present
|
echo/tierbtrace/tierbtrace_gbs8/trace_results.tsv
CHANGED
|
@@ -1,2 +1,3 @@
|
|
| 1 |
model backend capacity status report
|
| 2 |
QWEN3_235B echo 1.083984375 OK /workspace/tierb_study_20260826_144644/echo/tierbtrace/tierbtrace_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p083984375/nsys/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p083984375.nsys-rep
|
|
|
|
|
|
| 1 |
model backend capacity status report
|
| 2 |
QWEN3_235B echo 1.083984375 OK /workspace/tierb_study_20260826_144644/echo/tierbtrace/tierbtrace_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p083984375/nsys/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p083984375.nsys-rep
|
| 3 |
+
QWEN3_30B echo 1.11328125 OK /workspace/tierb_study_20260826_144644/echo/tierbtrace/tierbtrace_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125/nsys/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125.nsys-rep
|