AbhishekGhosh1998 commited on
Commit
23cfbc7
·
verified ·
1 Parent(s): ff9bef2

granularity results echo/tierbmem @ 2026-08-26T19:00:15Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 (part 3)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185717637255Z.pickle +3 -0
  2. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185721321980Z.json +79 -0
  3. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185721321980Z.pickle +3 -0
  4. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185721967094Z.json +79 -0
  5. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185721967094Z.pickle +3 -0
  6. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185722392904Z.json +79 -0
  7. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185722392904Z.pickle +3 -0
  8. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185714716388Z.json +79 -0
  9. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185714716388Z.pickle +3 -0
  10. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185716268525Z.json +79 -0
  11. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185716268525Z.pickle +3 -0
  12. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185717637230Z.json +79 -0
  13. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185717637230Z.pickle +3 -0
  14. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185721322058Z.json +79 -0
  15. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185721322058Z.pickle +3 -0
  16. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185721967116Z.json +79 -0
  17. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185721967116Z.pickle +3 -0
  18. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185722392867Z.json +79 -0
  19. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185722392867Z.pickle +3 -0
  20. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185714716387Z.json +79 -0
  21. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185714716387Z.pickle +3 -0
  22. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185716268545Z.json +79 -0
  23. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185716268545Z.pickle +3 -0
  24. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185717637239Z.json +79 -0
  25. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185717637239Z.pickle +3 -0
  26. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185721321994Z.json +79 -0
  27. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185721321994Z.pickle +3 -0
  28. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185721967103Z.json +79 -0
  29. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185721967103Z.pickle +3 -0
  30. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185722392876Z.json +79 -0
  31. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185722392876Z.pickle +3 -0
  32. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185714716388Z.json +79 -0
  33. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185714716388Z.pickle +3 -0
  34. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185716268537Z.json +79 -0
  35. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185716268537Z.pickle +3 -0
  36. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185717637252Z.json +79 -0
  37. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185717637252Z.pickle +3 -0
  38. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185721322186Z.json +79 -0
  39. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185721322186Z.pickle +3 -0
  40. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185721967126Z.json +79 -0
  41. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185721967126Z.pickle +3 -0
  42. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185722392843Z.json +79 -0
  43. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185722392843Z.pickle +3 -0
  44. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185714716374Z.json +79 -0
  45. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185714716374Z.pickle +3 -0
  46. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185716268523Z.json +79 -0
  47. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185716268523Z.pickle +3 -0
  48. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185717637212Z.json +79 -0
  49. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185717637212Z.pickle +3 -0
  50. echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185721322014Z.json +79 -0
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T185717637255Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0e1ecd78195dd2b51e2dcd4598c8defa74ed9b7ae05ef2be9f31a36339e085ba
3
+ size 9942271
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185721321980Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84649044992,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 129025703424,
5
+ "active_bytes_peak_delta": 6617244160,
6
+ "after": {
7
+ "active_bytes_current": 84649044992,
8
+ "active_bytes_peak": 129025703424,
9
+ "allocated_bytes_current": 84649044992,
10
+ "allocated_bytes_peak": 122408459264,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 4209382400,
13
+ "inactive_split_bytes_peak": 13272840192,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 142709096448,
21
+ "reserved_bytes_peak": 142709096448,
22
+ "segment_count_current": 3356,
23
+ "segment_count_peak": 3356,
24
+ "timestamp_utc": "2026-08-26T18:57:21.321469Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84649044992,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 122408459264,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 84632001536,
33
+ "active_bytes_peak": 122408459264,
34
+ "allocated_bytes_current": 84632001536,
35
+ "allocated_bytes_peak": 122408459264,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 4173997056,
38
+ "inactive_split_bytes_peak": 12200414208,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 123394326528,
46
+ "reserved_bytes_peak": 123394326528,
47
+ "segment_count_current": 2923,
48
+ "segment_count_peak": 2923,
49
+ "timestamp_utc": "2026-08-26T18:57:17.862485Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4209382400,
55
+ "inactive_split_bytes_current_delta": 35385344,
56
+ "inactive_split_bytes_peak": 13272840192,
57
+ "inactive_split_bytes_peak_delta": 1072425984,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 142709096448,
68
+ "reserved_bytes_current_delta": 19314769920,
69
+ "reserved_bytes_peak": 142709096448,
70
+ "reserved_bytes_peak_delta": 19314769920,
71
+ "segment_count_current": 3356,
72
+ "segment_count_current_delta": 433,
73
+ "segment_count_peak": 3356,
74
+ "segment_count_peak_delta": 433,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T185721321980Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:21.536837Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T185721321980Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dd3ad7429d7eeaad8c56aa12c5c1898f9108ed6966ff5f03b7d59c26d6891993
3
+ size 13044981
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185721967094Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84649044992,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 129025703424,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 84649044992,
8
+ "active_bytes_peak": 129025703424,
9
+ "allocated_bytes_current": 84649044992,
10
+ "allocated_bytes_peak": 122408459264,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 4209382400,
13
+ "inactive_split_bytes_peak": 13272840192,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 142709096448,
21
+ "reserved_bytes_peak": 142709096448,
22
+ "segment_count_current": 3356,
23
+ "segment_count_peak": 3356,
24
+ "timestamp_utc": "2026-08-26T18:57:21.966678Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84649044992,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 122408459264,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 84649044992,
33
+ "active_bytes_peak": 129025703424,
34
+ "allocated_bytes_current": 84649044992,
35
+ "allocated_bytes_peak": 122408459264,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 4209382400,
38
+ "inactive_split_bytes_peak": 13272840192,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 142709096448,
46
+ "reserved_bytes_peak": 142709096448,
47
+ "segment_count_current": 3356,
48
+ "segment_count_peak": 3356,
49
+ "timestamp_utc": "2026-08-26T18:57:21.565839Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4209382400,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 13272840192,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 142709096448,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 142709096448,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 3356,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 3356,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T185721967094Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:21.991948Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T185721967094Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:11bace237037948288f21ac0d282aa1371521ae5a556b3a40f8884d69666e01d
3
+ size 1147818
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185722392904Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84649044992,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 129025703424,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 84649044992,
8
+ "active_bytes_peak": 129025703424,
9
+ "allocated_bytes_current": 84649044992,
10
+ "allocated_bytes_peak": 122408459264,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 4209382400,
13
+ "inactive_split_bytes_peak": 13272840192,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 142709096448,
21
+ "reserved_bytes_peak": 142709096448,
22
+ "segment_count_current": 3356,
23
+ "segment_count_peak": 3356,
24
+ "timestamp_utc": "2026-08-26T18:57:22.392483Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84649044992,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 122408459264,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 84649044992,
33
+ "active_bytes_peak": 129025703424,
34
+ "allocated_bytes_current": 84649044992,
35
+ "allocated_bytes_peak": 122408459264,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 4209382400,
38
+ "inactive_split_bytes_peak": 13272840192,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 142709096448,
46
+ "reserved_bytes_peak": 142709096448,
47
+ "segment_count_current": 3356,
48
+ "segment_count_peak": 3356,
49
+ "timestamp_utc": "2026-08-26T18:57:21.993666Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4209382400,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 13272840192,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 142709096448,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 142709096448,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 3356,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 3356,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T185722392904Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:22.416360Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T185722392904Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:80d60f38ec3abfba66f7d751f65e7bb4c0db57b61be034f1c8dda382a731d4b0
3
+ size 1147818
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185714716388Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84623316992,
3
+ "active_bytes_current_delta": 41069773824,
4
+ "active_bytes_peak": 91617038336,
5
+ "active_bytes_peak_delta": 48057203712,
6
+ "after": {
7
+ "active_bytes_current": 84623316992,
8
+ "active_bytes_peak": 91617038336,
9
+ "allocated_bytes_current": 84623316992,
10
+ "allocated_bytes_peak": 91617038336,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 4312705024,
13
+ "inactive_split_bytes_peak": 12182193664,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 93296001024,
21
+ "reserved_bytes_peak": 93296001024,
22
+ "segment_count_current": 2187,
23
+ "segment_count_peak": 2187,
24
+ "timestamp_utc": "2026-08-26T18:57:14.715878Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84623316992,
28
+ "allocated_bytes_current_delta": 41069773824,
29
+ "allocated_bytes_peak": 91617038336,
30
+ "allocated_bytes_peak_delta": 48057203712,
31
+ "before": {
32
+ "active_bytes_current": 43553543168,
33
+ "active_bytes_peak": 43559834624,
34
+ "allocated_bytes_current": 43553543168,
35
+ "allocated_bytes_peak": 43559834624,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 1256303616,
38
+ "inactive_split_bytes_peak": 1612912640,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 44835012608,
46
+ "reserved_bytes_peak": 44835012608,
47
+ "segment_count_current": 883,
48
+ "segment_count_peak": 883,
49
+ "timestamp_utc": "2026-08-26T18:56:36.124259Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4312705024,
55
+ "inactive_split_bytes_current_delta": 3056401408,
56
+ "inactive_split_bytes_peak": 12182193664,
57
+ "inactive_split_bytes_peak_delta": 10569281024,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 93296001024,
68
+ "reserved_bytes_current_delta": 48460988416,
69
+ "reserved_bytes_peak": 93296001024,
70
+ "reserved_bytes_peak_delta": 48460988416,
71
+ "segment_count_current": 2187,
72
+ "segment_count_current_delta": 1304,
73
+ "segment_count_peak": 2187,
74
+ "segment_count_peak_delta": 1304,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T185714716388Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:14.902087Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T185714716388Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b37c5ff79eea33f8e00c37e16a0c30121fae013bda203537694c76d3dd26ef1d
3
+ size 9776530
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185716268525Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84623316992,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 122401423360,
5
+ "active_bytes_peak_delta": 30784385024,
6
+ "after": {
7
+ "active_bytes_current": 84623316992,
8
+ "active_bytes_peak": 122401423360,
9
+ "allocated_bytes_current": 84623316992,
10
+ "allocated_bytes_peak": 122401423360,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 4214138880,
13
+ "inactive_split_bytes_peak": 12182193664,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 124092678144,
21
+ "reserved_bytes_peak": 124092678144,
22
+ "segment_count_current": 2896,
23
+ "segment_count_peak": 2896,
24
+ "timestamp_utc": "2026-08-26T18:57:16.268085Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84623316992,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 122401423360,
30
+ "allocated_bytes_peak_delta": 30784385024,
31
+ "before": {
32
+ "active_bytes_current": 84623316992,
33
+ "active_bytes_peak": 91617038336,
34
+ "allocated_bytes_current": 84623316992,
35
+ "allocated_bytes_peak": 91617038336,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 4312705024,
38
+ "inactive_split_bytes_peak": 12182193664,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 93296001024,
46
+ "reserved_bytes_peak": 93296001024,
47
+ "segment_count_current": 2187,
48
+ "segment_count_peak": 2187,
49
+ "timestamp_utc": "2026-08-26T18:57:14.936634Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4214138880,
55
+ "inactive_split_bytes_current_delta": -98566144,
56
+ "inactive_split_bytes_peak": 12182193664,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 124092678144,
68
+ "reserved_bytes_current_delta": 30796677120,
69
+ "reserved_bytes_peak": 124092678144,
70
+ "reserved_bytes_peak_delta": 30796677120,
71
+ "segment_count_current": 2896,
72
+ "segment_count_current_delta": 709,
73
+ "segment_count_peak": 2896,
74
+ "segment_count_peak_delta": 709,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T185716268525Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:16.452869Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T185716268525Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0ed47c8b35e9481a7efdee5e8b9467741a891e79a9557961f971e65adc8030b
3
+ size 9503132
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185717637230Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84622792704,
3
+ "active_bytes_current_delta": -524288,
4
+ "active_bytes_peak": 122862320640,
5
+ "active_bytes_peak_delta": 460897280,
6
+ "after": {
7
+ "active_bytes_current": 84622792704,
8
+ "active_bytes_peak": 122862320640,
9
+ "allocated_bytes_current": 84622792704,
10
+ "allocated_bytes_peak": 122409335808,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 4418086912,
13
+ "inactive_split_bytes_peak": 12182193664,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 124092678144,
21
+ "reserved_bytes_peak": 124092678144,
22
+ "segment_count_current": 2896,
23
+ "segment_count_peak": 2896,
24
+ "timestamp_utc": "2026-08-26T18:57:17.636808Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84622792704,
28
+ "allocated_bytes_current_delta": -524288,
29
+ "allocated_bytes_peak": 122409335808,
30
+ "allocated_bytes_peak_delta": 7912448,
31
+ "before": {
32
+ "active_bytes_current": 84623316992,
33
+ "active_bytes_peak": 122401423360,
34
+ "allocated_bytes_current": 84623316992,
35
+ "allocated_bytes_peak": 122401423360,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 4214138880,
38
+ "inactive_split_bytes_peak": 12182193664,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 124092678144,
46
+ "reserved_bytes_peak": 124092678144,
47
+ "segment_count_current": 2896,
48
+ "segment_count_peak": 2896,
49
+ "timestamp_utc": "2026-08-26T18:57:16.477699Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4418086912,
55
+ "inactive_split_bytes_current_delta": 203948032,
56
+ "inactive_split_bytes_peak": 12182193664,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 124092678144,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 124092678144,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 2896,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2896,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T185717637230Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:17.824121Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T185717637230Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dc955e4105c6e2315399dc8ef2b4a81962c2f9b2d7105e2abfd4462d12b09629
3
+ size 9479267
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185721322058Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84640360448,
3
+ "active_bytes_current_delta": 17567744,
4
+ "active_bytes_peak": 129017018880,
5
+ "active_bytes_peak_delta": 6154698240,
6
+ "after": {
7
+ "active_bytes_current": 84640360448,
8
+ "active_bytes_peak": 129017018880,
9
+ "allocated_bytes_current": 84640360448,
10
+ "allocated_bytes_peak": 122409335808,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 4350187520,
13
+ "inactive_split_bytes_peak": 13315079168,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 142943977472,
21
+ "reserved_bytes_peak": 142943977472,
22
+ "segment_count_current": 3336,
23
+ "segment_count_peak": 3336,
24
+ "timestamp_utc": "2026-08-26T18:57:21.321537Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84640360448,
28
+ "allocated_bytes_current_delta": 17567744,
29
+ "allocated_bytes_peak": 122409335808,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 84622792704,
33
+ "active_bytes_peak": 122862320640,
34
+ "allocated_bytes_current": 84622792704,
35
+ "allocated_bytes_peak": 122409335808,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 4418086912,
38
+ "inactive_split_bytes_peak": 12182193664,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 124092678144,
46
+ "reserved_bytes_peak": 124092678144,
47
+ "segment_count_current": 2896,
48
+ "segment_count_peak": 2896,
49
+ "timestamp_utc": "2026-08-26T18:57:17.848657Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4350187520,
55
+ "inactive_split_bytes_current_delta": -67899392,
56
+ "inactive_split_bytes_peak": 13315079168,
57
+ "inactive_split_bytes_peak_delta": 1132885504,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 142943977472,
68
+ "reserved_bytes_current_delta": 18851299328,
69
+ "reserved_bytes_peak": 142943977472,
70
+ "reserved_bytes_peak_delta": 18851299328,
71
+ "segment_count_current": 3336,
72
+ "segment_count_current_delta": 440,
73
+ "segment_count_peak": 3336,
74
+ "segment_count_peak_delta": 440,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T185721322058Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:21.523444Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T185721322058Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e81a30934d574b230fd9fcb7a0b0a469de4855c57b57d22ecb26f7ff3030996
3
+ size 11659206
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185721967116Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84640360448,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 129017018880,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 84640360448,
8
+ "active_bytes_peak": 129017018880,
9
+ "allocated_bytes_current": 84640360448,
10
+ "allocated_bytes_peak": 122409335808,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 4350187520,
13
+ "inactive_split_bytes_peak": 13315079168,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 142943977472,
21
+ "reserved_bytes_peak": 142943977472,
22
+ "segment_count_current": 3336,
23
+ "segment_count_peak": 3336,
24
+ "timestamp_utc": "2026-08-26T18:57:21.966674Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84640360448,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 122409335808,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 84640360448,
33
+ "active_bytes_peak": 129017018880,
34
+ "allocated_bytes_current": 84640360448,
35
+ "allocated_bytes_peak": 122409335808,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 4350187520,
38
+ "inactive_split_bytes_peak": 13315079168,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 142943977472,
46
+ "reserved_bytes_peak": 142943977472,
47
+ "segment_count_current": 3336,
48
+ "segment_count_peak": 3336,
49
+ "timestamp_utc": "2026-08-26T18:57:21.550555Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4350187520,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 13315079168,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 142943977472,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 142943977472,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 3336,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 3336,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T185721967116Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:21.990538Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T185721967116Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d065d6e7a7a58cce740e1eb2653e0a7fb3332951ac483721044a3a3280b0af5c
3
+ size 1024383
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185722392867Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84640360448,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 129017018880,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 84640360448,
8
+ "active_bytes_peak": 129017018880,
9
+ "allocated_bytes_current": 84640360448,
10
+ "allocated_bytes_peak": 122409335808,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 4350187520,
13
+ "inactive_split_bytes_peak": 13315079168,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 142943977472,
21
+ "reserved_bytes_peak": 142943977472,
22
+ "segment_count_current": 3336,
23
+ "segment_count_peak": 3336,
24
+ "timestamp_utc": "2026-08-26T18:57:22.392470Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84640360448,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 122409335808,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 84640360448,
33
+ "active_bytes_peak": 129017018880,
34
+ "allocated_bytes_current": 84640360448,
35
+ "allocated_bytes_peak": 122409335808,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 4350187520,
38
+ "inactive_split_bytes_peak": 13315079168,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 142943977472,
46
+ "reserved_bytes_peak": 142943977472,
47
+ "segment_count_current": 3336,
48
+ "segment_count_peak": 3336,
49
+ "timestamp_utc": "2026-08-26T18:57:21.992237Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4350187520,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 13315079168,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 142943977472,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 142943977472,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 3336,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 3336,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T185722392867Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:22.413897Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T185722392867Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0d40a12d5ca3f1a42a5aeb79a48e78cc0e6b77a16c8a8a15c856f8f778fc1552
3
+ size 1024414
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185714716387Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84639218688,
3
+ "active_bytes_current_delta": 41082396160,
4
+ "active_bytes_peak": 91614024192,
5
+ "active_bytes_peak_delta": 48050910208,
6
+ "after": {
7
+ "active_bytes_current": 84639218688,
8
+ "active_bytes_peak": 91614024192,
9
+ "allocated_bytes_current": 84639218688,
10
+ "allocated_bytes_peak": 91614024192,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 4003202048,
13
+ "inactive_split_bytes_peak": 12193790976,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 93308583936,
21
+ "reserved_bytes_peak": 93308583936,
22
+ "segment_count_current": 2209,
23
+ "segment_count_peak": 2209,
24
+ "timestamp_utc": "2026-08-26T18:57:14.715878Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84639218688,
28
+ "allocated_bytes_current_delta": 41082396160,
29
+ "allocated_bytes_peak": 91614024192,
30
+ "allocated_bytes_peak_delta": 48050910208,
31
+ "before": {
32
+ "active_bytes_current": 43556822528,
33
+ "active_bytes_peak": 43563113984,
34
+ "allocated_bytes_current": 43556822528,
35
+ "allocated_bytes_peak": 43563113984,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 1223664128,
38
+ "inactive_split_bytes_peak": 1612912640,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 44795166720,
46
+ "reserved_bytes_peak": 44795166720,
47
+ "segment_count_current": 887,
48
+ "segment_count_peak": 887,
49
+ "timestamp_utc": "2026-08-26T18:56:36.124364Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4003202048,
55
+ "inactive_split_bytes_current_delta": 2779537920,
56
+ "inactive_split_bytes_peak": 12193790976,
57
+ "inactive_split_bytes_peak_delta": 10580878336,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 93308583936,
68
+ "reserved_bytes_current_delta": 48513417216,
69
+ "reserved_bytes_peak": 93308583936,
70
+ "reserved_bytes_peak_delta": 48513417216,
71
+ "segment_count_current": 2209,
72
+ "segment_count_current_delta": 1322,
73
+ "segment_count_peak": 2209,
74
+ "segment_count_peak_delta": 1322,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T185714716387Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:14.911068Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T185714716387Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8bbc327b800ea28c61c5b117e64d564f9029b195268fced181206659829598e5
3
+ size 11336704
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185716268545Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84639218688,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 122380765184,
5
+ "active_bytes_peak_delta": 30766740992,
6
+ "after": {
7
+ "active_bytes_current": 84639218688,
8
+ "active_bytes_peak": 122380765184,
9
+ "allocated_bytes_current": 84639218688,
10
+ "allocated_bytes_peak": 122380765184,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 3904635904,
13
+ "inactive_split_bytes_peak": 12193790976,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 124080095232,
21
+ "reserved_bytes_peak": 124080095232,
22
+ "segment_count_current": 2932,
23
+ "segment_count_peak": 2932,
24
+ "timestamp_utc": "2026-08-26T18:57:16.268107Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84639218688,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 122380765184,
30
+ "allocated_bytes_peak_delta": 30766740992,
31
+ "before": {
32
+ "active_bytes_current": 84639218688,
33
+ "active_bytes_peak": 91614024192,
34
+ "allocated_bytes_current": 84639218688,
35
+ "allocated_bytes_peak": 91614024192,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 4003202048,
38
+ "inactive_split_bytes_peak": 12193790976,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 93308583936,
46
+ "reserved_bytes_peak": 93308583936,
47
+ "segment_count_current": 2209,
48
+ "segment_count_peak": 2209,
49
+ "timestamp_utc": "2026-08-26T18:57:14.949036Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 3904635904,
55
+ "inactive_split_bytes_current_delta": -98566144,
56
+ "inactive_split_bytes_peak": 12193790976,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 124080095232,
68
+ "reserved_bytes_current_delta": 30771511296,
69
+ "reserved_bytes_peak": 124080095232,
70
+ "reserved_bytes_peak_delta": 30771511296,
71
+ "segment_count_current": 2932,
72
+ "segment_count_current_delta": 723,
73
+ "segment_count_peak": 2932,
74
+ "segment_count_peak_delta": 723,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T185716268545Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:16.465288Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T185716268545Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f24e01ae2a41576be2f51d98cdcca72d4e4107f29c84218a35eeeace42684f1e
3
+ size 10005096
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185717637239Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84639218688,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 122385467392,
5
+ "active_bytes_peak_delta": 4702208,
6
+ "after": {
7
+ "active_bytes_current": 84639218688,
8
+ "active_bytes_peak": 122385467392,
9
+ "allocated_bytes_current": 84639218688,
10
+ "allocated_bytes_peak": 122385467392,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 4007396352,
13
+ "inactive_split_bytes_peak": 12193790976,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 124080095232,
21
+ "reserved_bytes_peak": 124080095232,
22
+ "segment_count_current": 2932,
23
+ "segment_count_peak": 2932,
24
+ "timestamp_utc": "2026-08-26T18:57:17.636816Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84639218688,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 122385467392,
30
+ "allocated_bytes_peak_delta": 4702208,
31
+ "before": {
32
+ "active_bytes_current": 84639218688,
33
+ "active_bytes_peak": 122380765184,
34
+ "allocated_bytes_current": 84639218688,
35
+ "allocated_bytes_peak": 122380765184,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 3904635904,
38
+ "inactive_split_bytes_peak": 12193790976,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 124080095232,
46
+ "reserved_bytes_peak": 124080095232,
47
+ "segment_count_current": 2932,
48
+ "segment_count_peak": 2932,
49
+ "timestamp_utc": "2026-08-26T18:57:16.493482Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4007396352,
55
+ "inactive_split_bytes_current_delta": 102760448,
56
+ "inactive_split_bytes_peak": 12193790976,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 124080095232,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 124080095232,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 2932,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2932,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T185717637239Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:17.835543Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T185717637239Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bf995b39791b738cb9918361bf3dfeabbdb7755842c029e8c32e0679685a6dd1
3
+ size 9983320
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185721321994Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84656262144,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 129032920576,
5
+ "active_bytes_peak_delta": 6647453184,
6
+ "after": {
7
+ "active_bytes_current": 84656262144,
8
+ "active_bytes_peak": 129032920576,
9
+ "allocated_bytes_current": 84656262144,
10
+ "allocated_bytes_peak": 122385467392,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 4042781696,
13
+ "inactive_split_bytes_peak": 13104142336,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 142549712896,
21
+ "reserved_bytes_peak": 142549712896,
22
+ "segment_count_current": 3353,
23
+ "segment_count_peak": 3353,
24
+ "timestamp_utc": "2026-08-26T18:57:21.321480Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84656262144,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 122385467392,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 84639218688,
33
+ "active_bytes_peak": 122385467392,
34
+ "allocated_bytes_current": 84639218688,
35
+ "allocated_bytes_peak": 122385467392,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 4007396352,
38
+ "inactive_split_bytes_peak": 12193790976,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 124080095232,
46
+ "reserved_bytes_peak": 124080095232,
47
+ "segment_count_current": 2932,
48
+ "segment_count_peak": 2932,
49
+ "timestamp_utc": "2026-08-26T18:57:17.862945Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4042781696,
55
+ "inactive_split_bytes_current_delta": 35385344,
56
+ "inactive_split_bytes_peak": 13104142336,
57
+ "inactive_split_bytes_peak_delta": 910351360,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 142549712896,
68
+ "reserved_bytes_current_delta": 18469617664,
69
+ "reserved_bytes_peak": 142549712896,
70
+ "reserved_bytes_peak_delta": 18469617664,
71
+ "segment_count_current": 3353,
72
+ "segment_count_current_delta": 421,
73
+ "segment_count_peak": 3353,
74
+ "segment_count_peak_delta": 421,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T185721321994Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:21.533677Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T185721321994Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:40569b1347f21a5e50f7a875558837dc21417d48ee9975658ac7638eb7a0952f
3
+ size 13043697
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185721967103Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84656262144,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 129032920576,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 84656262144,
8
+ "active_bytes_peak": 129032920576,
9
+ "allocated_bytes_current": 84656262144,
10
+ "allocated_bytes_peak": 122385467392,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 4042781696,
13
+ "inactive_split_bytes_peak": 13104142336,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 142549712896,
21
+ "reserved_bytes_peak": 142549712896,
22
+ "segment_count_current": 3353,
23
+ "segment_count_peak": 3353,
24
+ "timestamp_utc": "2026-08-26T18:57:21.966676Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84656262144,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 122385467392,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 84656262144,
33
+ "active_bytes_peak": 129032920576,
34
+ "allocated_bytes_current": 84656262144,
35
+ "allocated_bytes_peak": 122385467392,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 4042781696,
38
+ "inactive_split_bytes_peak": 13104142336,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 142549712896,
46
+ "reserved_bytes_peak": 142549712896,
47
+ "segment_count_current": 3353,
48
+ "segment_count_peak": 3353,
49
+ "timestamp_utc": "2026-08-26T18:57:21.562190Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4042781696,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 13104142336,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 142549712896,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 142549712896,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 3353,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 3353,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T185721967103Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:21.991954Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T185721967103Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b05b8fd9cfb442a68ac108921b2a52dc9cb45f3ace93957b3d1a37262674f1d4
3
+ size 1145785
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185722392876Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84656262144,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 129032920576,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 84656262144,
8
+ "active_bytes_peak": 129032920576,
9
+ "allocated_bytes_current": 84656262144,
10
+ "allocated_bytes_peak": 122385467392,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 4042781696,
13
+ "inactive_split_bytes_peak": 13104142336,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 142549712896,
21
+ "reserved_bytes_peak": 142549712896,
22
+ "segment_count_current": 3353,
23
+ "segment_count_peak": 3353,
24
+ "timestamp_utc": "2026-08-26T18:57:22.392480Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84656262144,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 122385467392,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 84656262144,
33
+ "active_bytes_peak": 129032920576,
34
+ "allocated_bytes_current": 84656262144,
35
+ "allocated_bytes_peak": 122385467392,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 4042781696,
38
+ "inactive_split_bytes_peak": 13104142336,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 142549712896,
46
+ "reserved_bytes_peak": 142549712896,
47
+ "segment_count_current": 3353,
48
+ "segment_count_peak": 3353,
49
+ "timestamp_utc": "2026-08-26T18:57:21.993694Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4042781696,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 13104142336,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 142549712896,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 142549712896,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 3353,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 3353,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T185722392876Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:22.415105Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T185722392876Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7f80a025f020d7c7fa17f8345eb5bb620908d07eea41fae6730f7901da0fd6d3
3
+ size 1145785
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185714716388Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84620231168,
3
+ "active_bytes_current_delta": 41069553664,
4
+ "active_bytes_peak": 91608373760,
5
+ "active_bytes_peak_delta": 48051404800,
6
+ "after": {
7
+ "active_bytes_current": 84620231168,
8
+ "active_bytes_peak": 91608373760,
9
+ "allocated_bytes_current": 84620231168,
10
+ "allocated_bytes_peak": 91608373760,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 4173184512,
13
+ "inactive_split_bytes_peak": 12389535744,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 93308583936,
21
+ "reserved_bytes_peak": 93308583936,
22
+ "segment_count_current": 2199,
23
+ "segment_count_peak": 2199,
24
+ "timestamp_utc": "2026-08-26T18:57:14.715883Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84620231168,
28
+ "allocated_bytes_current_delta": 41069553664,
29
+ "allocated_bytes_peak": 91608373760,
30
+ "allocated_bytes_peak_delta": 48051404800,
31
+ "before": {
32
+ "active_bytes_current": 43550677504,
33
+ "active_bytes_peak": 43556968960,
34
+ "allocated_bytes_current": 43550677504,
35
+ "allocated_bytes_peak": 43556968960,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 1252877824,
38
+ "inactive_split_bytes_peak": 1612912640,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 44820332544,
46
+ "reserved_bytes_peak": 44820332544,
47
+ "segment_count_current": 889,
48
+ "segment_count_peak": 889,
49
+ "timestamp_utc": "2026-08-26T18:56:36.124297Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4173184512,
55
+ "inactive_split_bytes_current_delta": 2920306688,
56
+ "inactive_split_bytes_peak": 12389535744,
57
+ "inactive_split_bytes_peak_delta": 10776623104,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 93308583936,
68
+ "reserved_bytes_current_delta": 48488251392,
69
+ "reserved_bytes_peak": 93308583936,
70
+ "reserved_bytes_peak_delta": 48488251392,
71
+ "segment_count_current": 2199,
72
+ "segment_count_current_delta": 1310,
73
+ "segment_count_peak": 2199,
74
+ "segment_count_peak_delta": 1310,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T185714716388Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:14.907724Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185714716388Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:808df123683f0e4d1437cfc18860e38f6b8f0e2a1b8f889185f461ff7d43a04a
3
+ size 10265593
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185716268537Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84620231168,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 122397165568,
5
+ "active_bytes_peak_delta": 30788791808,
6
+ "after": {
7
+ "active_bytes_current": 84620231168,
8
+ "active_bytes_peak": 122397165568,
9
+ "allocated_bytes_current": 84620231168,
10
+ "allocated_bytes_peak": 122397165568,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 4074618368,
13
+ "inactive_split_bytes_peak": 12389535744,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 124111552512,
21
+ "reserved_bytes_peak": 124111552512,
22
+ "segment_count_current": 2919,
23
+ "segment_count_peak": 2919,
24
+ "timestamp_utc": "2026-08-26T18:57:16.268101Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84620231168,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 122397165568,
30
+ "allocated_bytes_peak_delta": 30788791808,
31
+ "before": {
32
+ "active_bytes_current": 84620231168,
33
+ "active_bytes_peak": 91608373760,
34
+ "allocated_bytes_current": 84620231168,
35
+ "allocated_bytes_peak": 91608373760,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 4173184512,
38
+ "inactive_split_bytes_peak": 12389535744,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 93308583936,
46
+ "reserved_bytes_peak": 93308583936,
47
+ "segment_count_current": 2199,
48
+ "segment_count_peak": 2199,
49
+ "timestamp_utc": "2026-08-26T18:57:14.946209Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4074618368,
55
+ "inactive_split_bytes_current_delta": -98566144,
56
+ "inactive_split_bytes_peak": 12389535744,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 124111552512,
68
+ "reserved_bytes_current_delta": 30802968576,
69
+ "reserved_bytes_peak": 124111552512,
70
+ "reserved_bytes_peak_delta": 30802968576,
71
+ "segment_count_current": 2919,
72
+ "segment_count_current_delta": 720,
73
+ "segment_count_peak": 2919,
74
+ "segment_count_peak_delta": 720,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T185716268537Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:16.464169Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185716268537Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9546a3cdedc19d1379af2b642d67f7f7f9fe8f9d25155d6e6c41a6354f1d167e
3
+ size 9964097
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185717637252Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84620231168,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 122566591488,
5
+ "active_bytes_peak_delta": 169425920,
6
+ "after": {
7
+ "active_bytes_current": 84620231168,
8
+ "active_bytes_peak": 122566591488,
9
+ "allocated_bytes_current": 84620231168,
10
+ "allocated_bytes_peak": 122415596544,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 4177378816,
13
+ "inactive_split_bytes_peak": 12389535744,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 124111552512,
21
+ "reserved_bytes_peak": 124111552512,
22
+ "segment_count_current": 2919,
23
+ "segment_count_peak": 2919,
24
+ "timestamp_utc": "2026-08-26T18:57:17.636821Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84620231168,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 122415596544,
30
+ "allocated_bytes_peak_delta": 18430976,
31
+ "before": {
32
+ "active_bytes_current": 84620231168,
33
+ "active_bytes_peak": 122397165568,
34
+ "allocated_bytes_current": 84620231168,
35
+ "allocated_bytes_peak": 122397165568,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 4074618368,
38
+ "inactive_split_bytes_peak": 12389535744,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 124111552512,
46
+ "reserved_bytes_peak": 124111552512,
47
+ "segment_count_current": 2919,
48
+ "segment_count_peak": 2919,
49
+ "timestamp_utc": "2026-08-26T18:57:16.492021Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4177378816,
55
+ "inactive_split_bytes_current_delta": 102760448,
56
+ "inactive_split_bytes_peak": 12389535744,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 124111552512,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 124111552512,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 2919,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2919,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T185717637252Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:17.832101Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185717637252Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a80831a0dd49d6d99c7c3777f30280b56c64c598e12b9b44375e59e393488bdf
3
+ size 9942396
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185721322186Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84637274624,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 129013933056,
5
+ "active_bytes_peak_delta": 6447341568,
6
+ "after": {
7
+ "active_bytes_current": 84637274624,
8
+ "active_bytes_peak": 129013933056,
9
+ "allocated_bytes_current": 84637274624,
10
+ "allocated_bytes_peak": 122415596544,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 4212764160,
13
+ "inactive_split_bytes_peak": 13276221952,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 142700707840,
21
+ "reserved_bytes_peak": 142700707840,
22
+ "segment_count_current": 3344,
23
+ "segment_count_peak": 3344,
24
+ "timestamp_utc": "2026-08-26T18:57:21.321529Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84637274624,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 122415596544,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 84620231168,
33
+ "active_bytes_peak": 122566591488,
34
+ "allocated_bytes_current": 84620231168,
35
+ "allocated_bytes_peak": 122415596544,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 4177378816,
38
+ "inactive_split_bytes_peak": 12389535744,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 124111552512,
46
+ "reserved_bytes_peak": 124111552512,
47
+ "segment_count_current": 2919,
48
+ "segment_count_peak": 2919,
49
+ "timestamp_utc": "2026-08-26T18:57:17.859724Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4212764160,
55
+ "inactive_split_bytes_current_delta": 35385344,
56
+ "inactive_split_bytes_peak": 13276221952,
57
+ "inactive_split_bytes_peak_delta": 886686208,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 142700707840,
68
+ "reserved_bytes_current_delta": 18589155328,
69
+ "reserved_bytes_peak": 142700707840,
70
+ "reserved_bytes_peak_delta": 18589155328,
71
+ "segment_count_current": 3344,
72
+ "segment_count_current_delta": 425,
73
+ "segment_count_peak": 3344,
74
+ "segment_count_peak_delta": 425,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T185721322186Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:21.531953Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185721322186Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bbccb966f0bc7c697ff4f83958c98600e6bf894d49398b6d8004b29286261149
3
+ size 13044392
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185721967126Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84637274624,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 129013933056,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 84637274624,
8
+ "active_bytes_peak": 129013933056,
9
+ "allocated_bytes_current": 84637274624,
10
+ "allocated_bytes_peak": 122415596544,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 4212764160,
13
+ "inactive_split_bytes_peak": 13276221952,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 142700707840,
21
+ "reserved_bytes_peak": 142700707840,
22
+ "segment_count_current": 3344,
23
+ "segment_count_peak": 3344,
24
+ "timestamp_utc": "2026-08-26T18:57:21.966680Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84637274624,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 122415596544,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 84637274624,
33
+ "active_bytes_peak": 129013933056,
34
+ "allocated_bytes_current": 84637274624,
35
+ "allocated_bytes_peak": 122415596544,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 4212764160,
38
+ "inactive_split_bytes_peak": 13276221952,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 142700707840,
46
+ "reserved_bytes_peak": 142700707840,
47
+ "segment_count_current": 3344,
48
+ "segment_count_peak": 3344,
49
+ "timestamp_utc": "2026-08-26T18:57:21.560891Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4212764160,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 13276221952,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 142700707840,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 142700707840,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 3344,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 3344,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T185721967126Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:21.992261Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185721967126Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:32ca825932a591126c68aa69b765727b43e77abb7ea5e931820e0d7e6a33c27e
3
+ size 1146117
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185722392843Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84637274624,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 129013933056,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 84637274624,
8
+ "active_bytes_peak": 129013933056,
9
+ "allocated_bytes_current": 84637274624,
10
+ "allocated_bytes_peak": 122415596544,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 4212764160,
13
+ "inactive_split_bytes_peak": 13276221952,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 142700707840,
21
+ "reserved_bytes_peak": 142700707840,
22
+ "segment_count_current": 3344,
23
+ "segment_count_peak": 3344,
24
+ "timestamp_utc": "2026-08-26T18:57:22.392443Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84637274624,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 122415596544,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 84637274624,
33
+ "active_bytes_peak": 129013933056,
34
+ "allocated_bytes_current": 84637274624,
35
+ "allocated_bytes_peak": 122415596544,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 4212764160,
38
+ "inactive_split_bytes_peak": 13276221952,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 142700707840,
46
+ "reserved_bytes_peak": 142700707840,
47
+ "segment_count_current": 3344,
48
+ "segment_count_peak": 3344,
49
+ "timestamp_utc": "2026-08-26T18:57:21.994010Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4212764160,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 13276221952,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 142700707840,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 142700707840,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 3344,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 3344,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T185722392843Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:22.415679Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185722392843Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4b4a812c8c9953fd12d61a9fa1a3c060b55cf99bd397c61e08fddd1e5740f5aa
3
+ size 1146117
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185714716374Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84623316992,
3
+ "active_bytes_current_delta": 41069773824,
4
+ "active_bytes_peak": 91617038336,
5
+ "active_bytes_peak_delta": 48057203712,
6
+ "after": {
7
+ "active_bytes_current": 84623316992,
8
+ "active_bytes_peak": 91617038336,
9
+ "allocated_bytes_current": 84623316992,
10
+ "allocated_bytes_peak": 91617038336,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 4312705024,
13
+ "inactive_split_bytes_peak": 12182193664,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 93296001024,
21
+ "reserved_bytes_peak": 93296001024,
22
+ "segment_count_current": 2187,
23
+ "segment_count_peak": 2187,
24
+ "timestamp_utc": "2026-08-26T18:57:14.715850Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84623316992,
28
+ "allocated_bytes_current_delta": 41069773824,
29
+ "allocated_bytes_peak": 91617038336,
30
+ "allocated_bytes_peak_delta": 48057203712,
31
+ "before": {
32
+ "active_bytes_current": 43553543168,
33
+ "active_bytes_peak": 43559834624,
34
+ "allocated_bytes_current": 43553543168,
35
+ "allocated_bytes_peak": 43559834624,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 1256303616,
38
+ "inactive_split_bytes_peak": 1612912640,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 44835012608,
46
+ "reserved_bytes_peak": 44835012608,
47
+ "segment_count_current": 883,
48
+ "segment_count_peak": 883,
49
+ "timestamp_utc": "2026-08-26T18:56:36.124260Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4312705024,
55
+ "inactive_split_bytes_current_delta": 3056401408,
56
+ "inactive_split_bytes_peak": 12182193664,
57
+ "inactive_split_bytes_peak_delta": 10569281024,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 93296001024,
68
+ "reserved_bytes_current_delta": 48460988416,
69
+ "reserved_bytes_peak": 93296001024,
70
+ "reserved_bytes_peak_delta": 48460988416,
71
+ "segment_count_current": 2187,
72
+ "segment_count_current_delta": 1304,
73
+ "segment_count_peak": 2187,
74
+ "segment_count_peak_delta": 1304,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T185714716374Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:14.907500Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185714716374Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aed1b5fc9e78f2e9ca4e6fc8395b8a81c19d4cf802bf1f36ef818da1c4d22d8d
3
+ size 9776530
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185716268523Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84623316992,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 122552418304,
5
+ "active_bytes_peak_delta": 30935379968,
6
+ "after": {
7
+ "active_bytes_current": 84623316992,
8
+ "active_bytes_peak": 122552418304,
9
+ "allocated_bytes_current": 84623316992,
10
+ "allocated_bytes_peak": 122401423360,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 4314802176,
13
+ "inactive_split_bytes_peak": 12182193664,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 124092678144,
21
+ "reserved_bytes_peak": 124092678144,
22
+ "segment_count_current": 2896,
23
+ "segment_count_peak": 2896,
24
+ "timestamp_utc": "2026-08-26T18:57:16.268085Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84623316992,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 122401423360,
30
+ "allocated_bytes_peak_delta": 30784385024,
31
+ "before": {
32
+ "active_bytes_current": 84623316992,
33
+ "active_bytes_peak": 91617038336,
34
+ "allocated_bytes_current": 84623316992,
35
+ "allocated_bytes_peak": 91617038336,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 4312705024,
38
+ "inactive_split_bytes_peak": 12182193664,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 93296001024,
46
+ "reserved_bytes_peak": 93296001024,
47
+ "segment_count_current": 2187,
48
+ "segment_count_peak": 2187,
49
+ "timestamp_utc": "2026-08-26T18:57:14.941036Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4314802176,
55
+ "inactive_split_bytes_current_delta": 2097152,
56
+ "inactive_split_bytes_peak": 12182193664,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 124092678144,
68
+ "reserved_bytes_current_delta": 30796677120,
69
+ "reserved_bytes_peak": 124092678144,
70
+ "reserved_bytes_peak_delta": 30796677120,
71
+ "segment_count_current": 2896,
72
+ "segment_count_current_delta": 709,
73
+ "segment_count_peak": 2896,
74
+ "segment_count_peak_delta": 709,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T185716268523Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:16.452271Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185716268523Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dbbeb24eacfb24bdbd734c5ff0663e0aacb114568394a159437e97c2de5818cb
3
+ size 9503153
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185717637212Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84623316992,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 122862844928,
5
+ "active_bytes_peak_delta": 310426624,
6
+ "after": {
7
+ "active_bytes_current": 84623316992,
8
+ "active_bytes_peak": 122862844928,
9
+ "allocated_bytes_current": 84623316992,
10
+ "allocated_bytes_peak": 122409860096,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 4518225920,
13
+ "inactive_split_bytes_peak": 12182193664,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 124092678144,
21
+ "reserved_bytes_peak": 124092678144,
22
+ "segment_count_current": 2896,
23
+ "segment_count_peak": 2896,
24
+ "timestamp_utc": "2026-08-26T18:57:17.636782Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84623316992,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 122409860096,
30
+ "allocated_bytes_peak_delta": 8436736,
31
+ "before": {
32
+ "active_bytes_current": 84623316992,
33
+ "active_bytes_peak": 122552418304,
34
+ "allocated_bytes_current": 84623316992,
35
+ "allocated_bytes_peak": 122401423360,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 4314802176,
38
+ "inactive_split_bytes_peak": 12182193664,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 124092678144,
46
+ "reserved_bytes_peak": 124092678144,
47
+ "segment_count_current": 2896,
48
+ "segment_count_peak": 2896,
49
+ "timestamp_utc": "2026-08-26T18:57:16.476772Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4518225920,
55
+ "inactive_split_bytes_current_delta": 203423744,
56
+ "inactive_split_bytes_peak": 12182193664,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 124092678144,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 124092678144,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 2896,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2896,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T185717637212Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:17.824669Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185717637212Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e53e1b1d1fd8cded3d575fbc8221f1e543225d075d6c45b4726535d8c3f70dad
3
+ size 9478730
echo/tierbmem/tierbmem_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185721322014Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 84640360448,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 129017018880,
5
+ "active_bytes_peak_delta": 6154173952,
6
+ "after": {
7
+ "active_bytes_current": 84640360448,
8
+ "active_bytes_peak": 129017018880,
9
+ "allocated_bytes_current": 84640360448,
10
+ "allocated_bytes_peak": 122409860096,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 4350187520,
13
+ "inactive_split_bytes_peak": 13315079168,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 143044640768,
21
+ "reserved_bytes_peak": 143044640768,
22
+ "segment_count_current": 3337,
23
+ "segment_count_peak": 3337,
24
+ "timestamp_utc": "2026-08-26T18:57:21.321493Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 84640360448,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 122409860096,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 84623316992,
33
+ "active_bytes_peak": 122862844928,
34
+ "allocated_bytes_current": 84623316992,
35
+ "allocated_bytes_peak": 122409860096,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 4518225920,
38
+ "inactive_split_bytes_peak": 12182193664,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 124092678144,
46
+ "reserved_bytes_peak": 124092678144,
47
+ "segment_count_current": 2896,
48
+ "segment_count_peak": 2896,
49
+ "timestamp_utc": "2026-08-26T18:57:17.848873Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 4350187520,
55
+ "inactive_split_bytes_current_delta": -168038400,
56
+ "inactive_split_bytes_peak": 13315079168,
57
+ "inactive_split_bytes_peak_delta": 1132885504,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 143044640768,
68
+ "reserved_bytes_current_delta": 18951962624,
69
+ "reserved_bytes_peak": 143044640768,
70
+ "reserved_bytes_peak_delta": 18951962624,
71
+ "segment_count_current": 3337,
72
+ "segment_count_current_delta": 441,
73
+ "segment_count_peak": 3337,
74
+ "segment_count_peak_delta": 441,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T185721322014Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:57:21.526139Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }