AbhishekGhosh1998 commited on
Commit
bec1cbd
·
verified ·
1 Parent(s): 6ea627c

granularity results granularity_next_b200/tiera_topk_trace/echo/tierbtrace @ 2026-09-17T03:22:24Z from hp-aiinfra-interactive-aghosh-8-xzthl-8xb200-moe-cg-0

Browse files
.gitattributes CHANGED
@@ -90,3 +90,4 @@ granularity_next_b200/tiera_topk_trace/echo/tierbtrace/tierbtrace_gbs16/G4_K4__e
90
  granularity_next_b200/tiera_topk_trace/echo/tierbtrace/tierbtrace_gbs16/G4_K6__echo__matched__seq4096__gbs16__cap1p1354166666666667/nsys/G4_K6__echo__matched__seq4096__gbs16__cap1p1354166666666667.nsys-rep filter=lfs diff=lfs merge=lfs -text
91
  granularity_next_b200/tiera_topk_trace/echo/tierbtrace/tierbtrace_gbs16/G4_K8__echo__matched__seq4096__gbs16__cap1p109375/nsys/G4_K8__echo__matched__seq4096__gbs16__cap1p109375.nsys-rep filter=lfs diff=lfs merge=lfs -text
92
  granularity_next_b200/tiera_topk_trace/echo/tierbtrace/tierbtrace_gbs16/G4_K10__echo__matched__seq4096__gbs16__cap1p1/nsys/G4_K10__echo__matched__seq4096__gbs16__cap1p1.nsys-rep filter=lfs diff=lfs merge=lfs -text
 
 
90
  granularity_next_b200/tiera_topk_trace/echo/tierbtrace/tierbtrace_gbs16/G4_K6__echo__matched__seq4096__gbs16__cap1p1354166666666667/nsys/G4_K6__echo__matched__seq4096__gbs16__cap1p1354166666666667.nsys-rep filter=lfs diff=lfs merge=lfs -text
91
  granularity_next_b200/tiera_topk_trace/echo/tierbtrace/tierbtrace_gbs16/G4_K8__echo__matched__seq4096__gbs16__cap1p109375/nsys/G4_K8__echo__matched__seq4096__gbs16__cap1p109375.nsys-rep filter=lfs diff=lfs merge=lfs -text
92
  granularity_next_b200/tiera_topk_trace/echo/tierbtrace/tierbtrace_gbs16/G4_K10__echo__matched__seq4096__gbs16__cap1p1/nsys/G4_K10__echo__matched__seq4096__gbs16__cap1p1.nsys-rep filter=lfs diff=lfs merge=lfs -text
93
+ granularity_next_b200/tiera_topk_trace/echo/tierbtrace/tierbtrace_gbs16/G4_K16__echo__matched__seq4096__gbs16__cap1p08203125/nsys/G4_K16__echo__matched__seq4096__gbs16__cap1p08203125.nsys-rep filter=lfs diff=lfs merge=lfs -text
granularity_next_b200/tiera_topk_trace/echo/tierbtrace/tierbtrace_gbs16/G4_K16__echo__matched__seq4096__gbs16__cap1p08203125/nsys/G4_K16__echo__matched__seq4096__gbs16__cap1p08203125.nsys-rep ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1eaa6341b3e9d20b72cd70231a44fd4449733db902ebaa83b28ae59225214896
3
+ size 20110548
granularity_next_b200/tiera_topk_trace/echo/tierbtrace/tierbtrace_gbs16/G4_K16__echo__matched__seq4096__gbs16__cap1p08203125/nsys/train.log CHANGED
@@ -1494,3 +1494,52 @@ CUDA graph capture done!!!
1494
  [2026-09-17 03:14:31] iteration 6/ 10 | consumed samples: 96 | elapsed time per iteration (ms): 392.7 | throughput per GPU (TFLOP/s/GPU): 496.2 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.012605E+01 | loss scale: 1.0 | grad norm: 15.290 | number of skipped iterations: 0 | number of nan iterations: 0 |
1495
  [2026-09-17 03:14:34] iteration 7/ 10 | consumed samples: 112 | elapsed time per iteration (ms): 3484.1 | throughput per GPU (TFLOP/s/GPU): 55.9 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.830610E+00 | loss scale: 1.0 | grad norm: 6.249 | number of skipped iterations: 0 | number of nan iterations: 0 |
1496
  [2026-09-17 03:14:35] iteration 8/ 10 | consumed samples: 128 | elapsed time per iteration (ms): 406.2 | throughput per GPU (TFLOP/s/GPU): 479.6 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.249561E+00 | loss scale: 1.0 | grad norm: 5.551 | number of skipped iterations: 0 | number of nan iterations: 0 |
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1494
  [2026-09-17 03:14:31] iteration 6/ 10 | consumed samples: 96 | elapsed time per iteration (ms): 392.7 | throughput per GPU (TFLOP/s/GPU): 496.2 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.012605E+01 | loss scale: 1.0 | grad norm: 15.290 | number of skipped iterations: 0 | number of nan iterations: 0 |
1495
  [2026-09-17 03:14:34] iteration 7/ 10 | consumed samples: 112 | elapsed time per iteration (ms): 3484.1 | throughput per GPU (TFLOP/s/GPU): 55.9 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.830610E+00 | loss scale: 1.0 | grad norm: 6.249 | number of skipped iterations: 0 | number of nan iterations: 0 |
1496
  [2026-09-17 03:14:35] iteration 8/ 10 | consumed samples: 128 | elapsed time per iteration (ms): 406.2 | throughput per GPU (TFLOP/s/GPU): 479.6 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.249561E+00 | loss scale: 1.0 | grad norm: 5.551 | number of skipped iterations: 0 | number of nan iterations: 0 |
1497
+ Capture range started in the application.
1498
+ Capture range ended in the application.
1499
+ Generating '/tmp/nsys-report-829b.qdstrm'
1500
+
1501
+
1502
+ Collection stop is not allowed in this state.
1503
+ W0917 03:22:05.734000 921743 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers
1504
+ W0917 03:22:05.734000 921743 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 921855 closing signal SIGTERM
1505
+ Generated:
1506
+ /workspace/granularity_next_b200/tiera_topk_trace/echo/tierbtrace/tierbtrace_gbs16/G4_K16__echo__matched__seq4096__gbs16__cap1p08203125/nsys/G4_K16__echo__matched__seq4096__gbs16__cap1p08203125.nsys-rep
1507
+ W0917 03:22:05.740000 921743 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 921856 closing signal SIGTERM
1508
+ W0917 03:22:05.742000 921743 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 921857 closing signal SIGTERM
1509
+ W0917 03:22:05.748000 921743 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 921858 closing signal SIGTERM
1510
+ W0917 03:22:05.750000 921743 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 921859 closing signal SIGTERM
1511
+ W0917 03:22:05.754000 921743 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 921860 closing signal SIGTERM
1512
+ W0917 03:22:05.769000 921743 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 921861 closing signal SIGTERM
1513
+ W0917 03:22:05.771000 921743 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 921862 closing signal SIGTERM
1514
+ Traceback (most recent call last):
1515
+ File "/usr/local/bin/torchrun", line 8, in <module>
1516
+ sys.exit(main())
1517
+ ^^^^^^
1518
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
1519
+ return f(*args, **kwargs)
1520
+ ^^^^^^^^^^^^^^^^^^
1521
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main
1522
+ run(args)
1523
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run
1524
+ elastic_launch(
1525
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__
1526
+ return launch_agent(self._config, self._entrypoint, list(args))
1527
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1528
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent
1529
+ result = agent.run()
1530
+ ^^^^^^^^^^^
1531
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
1532
+ result = f(*args, **kwargs)
1533
+ ^^^^^^^^^^^^^^^^^^
1534
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run
1535
+ result = self._invoke_run(role)
1536
+ ^^^^^^^^^^^^^^^^^^^^^^
1537
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run
1538
+ time.sleep(monitor_interval)
1539
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler
1540
+ raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
1541
+ torch.distributed.elastic.multiprocessing.api.SignalException: Process 921743 got signal: 15
1542
+ The target application terminated. One or more process it created re-parented.
1543
+ Waiting for termination of re-parented processes.
1544
+ Use the `--wait` option to modify this behavior.
1545
+ There are no active sessions.
granularity_next_b200/tiera_topk_trace/echo/tierbtrace/tierbtrace_gbs16/G4_K16__echo__matched__seq4096__gbs16__cap1p08203125/nsys/verify_trace_ranks.log ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ report : /workspace/granularity_next_b200/tiera_topk_trace/echo/tierbtrace/tierbtrace_gbs16/G4_K16__echo__matched__seq4096__gbs16__cap1p08203125/nsys/G4_K16__echo__matched__seq4096__gbs16__cap1p08203125.nsys-rep
2
+ kernels : 281286
3
+ devices : 8 of 8 -> [0, 1, 2, 3, 4, 5, 6, 7]
4
+ processes: 1
5
+ markers : 23552 echo_trace__ kernels on devices [0, 1, 2, 3, 4, 5, 6, 7]
6
+ PASS: all ranks present and backend markers emitted
granularity_next_b200/tiera_topk_trace/echo/tierbtrace/tierbtrace_gbs16/trace_results.tsv CHANGED
@@ -7,3 +7,4 @@ G4_K4 echo 1.171875 OK /workspace/granularity_next_b200/tiera_topk_trace/echo/ti
7
  G4_K6 echo 1.1354166666666667 OK /workspace/granularity_next_b200/tiera_topk_trace/echo/tierbtrace/tierbtrace_gbs16/G4_K6__echo__matched__seq4096__gbs16__cap1p1354166666666667/nsys/G4_K6__echo__matched__seq4096__gbs16__cap1p1354166666666667.nsys-rep
8
  G4_K8 echo 1.109375 OK /workspace/granularity_next_b200/tiera_topk_trace/echo/tierbtrace/tierbtrace_gbs16/G4_K8__echo__matched__seq4096__gbs16__cap1p109375/nsys/G4_K8__echo__matched__seq4096__gbs16__cap1p109375.nsys-rep
9
  G4_K10 echo 1.1 OK /workspace/granularity_next_b200/tiera_topk_trace/echo/tierbtrace/tierbtrace_gbs16/G4_K10__echo__matched__seq4096__gbs16__cap1p1/nsys/G4_K10__echo__matched__seq4096__gbs16__cap1p1.nsys-rep
 
 
7
  G4_K6 echo 1.1354166666666667 OK /workspace/granularity_next_b200/tiera_topk_trace/echo/tierbtrace/tierbtrace_gbs16/G4_K6__echo__matched__seq4096__gbs16__cap1p1354166666666667/nsys/G4_K6__echo__matched__seq4096__gbs16__cap1p1354166666666667.nsys-rep
8
  G4_K8 echo 1.109375 OK /workspace/granularity_next_b200/tiera_topk_trace/echo/tierbtrace/tierbtrace_gbs16/G4_K8__echo__matched__seq4096__gbs16__cap1p109375/nsys/G4_K8__echo__matched__seq4096__gbs16__cap1p109375.nsys-rep
9
  G4_K10 echo 1.1 OK /workspace/granularity_next_b200/tiera_topk_trace/echo/tierbtrace/tierbtrace_gbs16/G4_K10__echo__matched__seq4096__gbs16__cap1p1/nsys/G4_K10__echo__matched__seq4096__gbs16__cap1p1.nsys-rep
10
+ G4_K16 echo 1.08203125 OK /workspace/granularity_next_b200/tiera_topk_trace/echo/tierbtrace/tierbtrace_gbs16/G4_K16__echo__matched__seq4096__gbs16__cap1p08203125/nsys/G4_K16__echo__matched__seq4096__gbs16__cap1p08203125.nsys-rep