AbhishekGhosh1998 commited on
Commit
06ac8e4
·
verified ·
1 Parent(s): 9e2d613

granularity results echo/tierbtrace @ 2026-08-26T19:52:54Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0

Browse files
.gitattributes CHANGED
@@ -47,3 +47,4 @@ echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00
47
  echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log filter=lfs diff=lfs merge=lfs -text
48
  echo/tierbtrace/tierbtrace_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p083984375/nsys/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p083984375.nsys-rep filter=lfs diff=lfs merge=lfs -text
49
  echo/tierbtrace/tierbtrace_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125/nsys/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125.nsys-rep filter=lfs diff=lfs merge=lfs -text
 
 
47
  echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log filter=lfs diff=lfs merge=lfs -text
48
  echo/tierbtrace/tierbtrace_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p083984375/nsys/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p083984375.nsys-rep filter=lfs diff=lfs merge=lfs -text
49
  echo/tierbtrace/tierbtrace_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125/nsys/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125.nsys-rep filter=lfs diff=lfs merge=lfs -text
50
+ echo/tierbtrace/tierbtrace_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0755208333333333/nsys/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0755208333333333.nsys-rep filter=lfs diff=lfs merge=lfs -text
echo/tierbtrace/tierbtrace_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0755208333333333/nsys/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0755208333333333.nsys-rep ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bb0818748e309a82e483546035725dfb2b5a27d5507aa596a053474c7d87ac49
3
+ size 17103761
echo/tierbtrace/tierbtrace_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0755208333333333/nsys/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=echo
2
+ config=DSV2_LITE_L24
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=bracing-regions-through-kernels
6
+ commit=9a236793
7
+ num_layers=24
8
+ hidden=2048
9
+ ffn_hidden=10944
10
+ moe_ffn_hidden=1408
11
+ num_experts=64
12
+ topk=6
13
+ experts_per_rank=8
14
+ expert_params=8650752
15
+ heads=16
16
+ kv_groups=0
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=2816
20
+ shared_expert_overlap=0
21
+ mla=1
22
+ dense_first_layers=1
23
+ moe_layers=23
24
+ rotary_base=10000
25
+ rotary_scaling=40
26
+ qkv_bias=0
27
+ router_balancing=seq_aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=49152
39
+ mean_tokens_per_expert=6144
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=static
43
+ capacity=1.0755208333333333
44
+ cuda_graph=1
45
+ probe_mode=0
46
+ async_grad_reduce=0
47
+ branch_role=timing
48
+ memory_dump_mode=none
49
+ memory_snapshot_path=
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=10
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=1
60
+ kernel_trace=1
61
+ fp8_recipe=mxfp8
echo/tierbtrace/tierbtrace_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0755208333333333/nsys/train.log ADDED
The diff for this file is too large to render. See raw diff
 
echo/tierbtrace/tierbtrace_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125/nsys/train.log CHANGED
@@ -2803,3 +2803,47 @@ Capture range ended in the application.
2803
  Generating '/tmp/nsys-report-f6e5.qdstrm'
2804
 
2805
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2803
  Generating '/tmp/nsys-report-f6e5.qdstrm'
2804
 
2805
 
2806
+ Collection stop is not allowed in this state.
2807
+ W0826 19:46:13.770000 700305 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers
2808
+ W0826 19:46:13.771000 700305 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 700421 closing signal SIGTERM
2809
+ Generated:
2810
+ /workspace/tierb_study_20260826_144644/echo/tierbtrace/tierbtrace_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125/nsys/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125.nsys-rep
2811
+ W0826 19:46:13.771000 700305 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 700422 closing signal SIGTERM
2812
+ W0826 19:46:13.789000 700305 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 700423 closing signal SIGTERM
2813
+ W0826 19:46:13.812000 700305 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 700424 closing signal SIGTERM
2814
+ W0826 19:46:13.832000 700305 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 700425 closing signal SIGTERM
2815
+ W0826 19:46:13.841000 700305 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 700426 closing signal SIGTERM
2816
+ W0826 19:46:13.851000 700305 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 700427 closing signal SIGTERM
2817
+ W0826 19:46:13.866000 700305 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 700428 closing signal SIGTERM
2818
+ Traceback (most recent call last):
2819
+ File "/usr/local/bin/torchrun", line 8, in <module>
2820
+ sys.exit(main())
2821
+ ^^^^^^
2822
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
2823
+ return f(*args, **kwargs)
2824
+ ^^^^^^^^^^^^^^^^^^
2825
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main
2826
+ run(args)
2827
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run
2828
+ elastic_launch(
2829
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__
2830
+ return launch_agent(self._config, self._entrypoint, list(args))
2831
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
2832
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent
2833
+ result = agent.run()
2834
+ ^^^^^^^^^^^
2835
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
2836
+ result = f(*args, **kwargs)
2837
+ ^^^^^^^^^^^^^^^^^^
2838
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run
2839
+ result = self._invoke_run(role)
2840
+ ^^^^^^^^^^^^^^^^^^^^^^
2841
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run
2842
+ time.sleep(monitor_interval)
2843
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler
2844
+ raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
2845
+ torch.distributed.elastic.multiprocessing.api.SignalException: Process 700305 got signal: 15
2846
+ The target application terminated. One or more process it created re-parented.
2847
+ Waiting for termination of re-parented processes.
2848
+ Use the `--wait` option to modify this behavior.
2849
+ There are no active sessions.
echo/tierbtrace/tierbtrace_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125/nsys/verify_trace_ranks.log ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ report : /workspace/tierb_study_20260826_144644/echo/tierbtrace/tierbtrace_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125/nsys/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125.nsys-rep
2
+ kernels : 398480
3
+ devices : 8 of 8 -> [0, 1, 2, 3, 4, 5, 6, 7]
4
+ processes: 1
5
+ PASS: all ranks present
echo/tierbtrace/tierbtrace_gbs8/trace_results.tsv CHANGED
@@ -1,2 +1,3 @@
1
  model backend capacity status report
2
  QWEN3_235B echo 1.083984375 OK /workspace/tierb_study_20260826_144644/echo/tierbtrace/tierbtrace_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p083984375/nsys/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p083984375.nsys-rep
 
 
1
  model backend capacity status report
2
  QWEN3_235B echo 1.083984375 OK /workspace/tierb_study_20260826_144644/echo/tierbtrace/tierbtrace_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p083984375/nsys/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p083984375.nsys-rep
3
+ QWEN3_30B echo 1.11328125 OK /workspace/tierb_study_20260826_144644/echo/tierbtrace/tierbtrace_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125/nsys/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p11328125.nsys-rep