AbhishekGhosh1998 commited on
Commit
ff9bef2
·
verified ·
1 Parent(s): cd778d8

granularity results echo/tierbmem @ 2026-08-26T19:00:15Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 (part 2)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185313625657Z.json +79 -0
  2. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185313625657Z.pickle +3 -0
  3. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185314224423Z.json +79 -0
  4. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185314224423Z.pickle +3 -0
  5. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185314987788Z.json +79 -0
  6. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185314987788Z.pickle +3 -0
  7. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185317851532Z.json +79 -0
  8. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185317851532Z.pickle +3 -0
  9. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185318232463Z.json +79 -0
  10. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185318232463Z.pickle +3 -0
  11. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185318559260Z.json +79 -0
  12. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185318559260Z.pickle +3 -0
  13. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185313625641Z.json +79 -0
  14. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185313625641Z.pickle +3 -0
  15. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185314224386Z.json +79 -0
  16. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185314224386Z.pickle +3 -0
  17. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185314987781Z.json +79 -0
  18. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185314987781Z.pickle +3 -0
  19. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185317851541Z.json +79 -0
  20. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185317851541Z.pickle +3 -0
  21. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185318232528Z.json +79 -0
  22. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185318232528Z.pickle +3 -0
  23. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185318559265Z.json +79 -0
  24. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185318559265Z.pickle +3 -0
  25. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185313625630Z.json +79 -0
  26. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185313625630Z.pickle +3 -0
  27. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185314224359Z.json +79 -0
  28. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185314224359Z.pickle +3 -0
  29. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185314987697Z.json +79 -0
  30. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185314987697Z.pickle +3 -0
  31. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185317851455Z.json +79 -0
  32. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185317851455Z.pickle +3 -0
  33. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185318232431Z.json +79 -0
  34. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185318232431Z.pickle +3 -0
  35. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185318559244Z.json +79 -0
  36. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185318559244Z.pickle +3 -0
  37. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185313625648Z.json +79 -0
  38. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185313625648Z.pickle +3 -0
  39. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185314224486Z.json +79 -0
  40. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185314224486Z.pickle +3 -0
  41. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185314987802Z.json +79 -0
  42. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185314987802Z.pickle +3 -0
  43. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185317851617Z.json +79 -0
  44. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185317851617Z.pickle +3 -0
  45. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185318232528Z.json +79 -0
  46. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185318232528Z.pickle +3 -0
  47. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185318559322Z.json +79 -0
  48. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185318559322Z.pickle +3 -0
  49. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env +61 -0
  50. echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log +0 -0
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185313625657Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85305508352,
3
+ "active_bytes_current_delta": 40784898560,
4
+ "active_bytes_peak": 92964658688,
5
+ "active_bytes_peak_delta": 48418883072,
6
+ "after": {
7
+ "active_bytes_current": 85305508352,
8
+ "active_bytes_peak": 92964658688,
9
+ "allocated_bytes_current": 85305508352,
10
+ "allocated_bytes_peak": 92964658688,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 1738882560,
13
+ "inactive_split_bytes_peak": 11517025280,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 103718846464,
21
+ "reserved_bytes_peak": 103718846464,
22
+ "segment_count_current": 1773,
23
+ "segment_count_peak": 1773,
24
+ "timestamp_utc": "2026-08-26T18:53:13.624966Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85305508352,
28
+ "allocated_bytes_current_delta": 40784898560,
29
+ "allocated_bytes_peak": 92964658688,
30
+ "allocated_bytes_peak_delta": 48418883072,
31
+ "before": {
32
+ "active_bytes_current": 44520609792,
33
+ "active_bytes_peak": 44545775616,
34
+ "allocated_bytes_current": 44520609792,
35
+ "allocated_bytes_peak": 44545775616,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 270362624,
38
+ "inactive_split_bytes_peak": 874342400,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 47949283328,
46
+ "reserved_bytes_peak": 47949283328,
47
+ "segment_count_current": 615,
48
+ "segment_count_peak": 615,
49
+ "timestamp_utc": "2026-08-26T18:52:35.069417Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1738882560,
55
+ "inactive_split_bytes_current_delta": 1468519936,
56
+ "inactive_split_bytes_peak": 11517025280,
57
+ "inactive_split_bytes_peak_delta": 10642682880,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 103718846464,
68
+ "reserved_bytes_current_delta": 55769563136,
69
+ "reserved_bytes_peak": 103718846464,
70
+ "reserved_bytes_peak_delta": 55769563136,
71
+ "segment_count_current": 1773,
72
+ "segment_count_current_delta": 1158,
73
+ "segment_count_peak": 1773,
74
+ "segment_count_peak_delta": 1158,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T185313625657Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:13.678100Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T185313625657Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:694ea38ba10338075e96eeda159409303d08f01f3ce6b6542de81a88c56d0973
3
+ size 3143234
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185314224423Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85305508352,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 123848197632,
5
+ "active_bytes_peak_delta": 30883538944,
6
+ "after": {
7
+ "active_bytes_current": 85305508352,
8
+ "active_bytes_peak": 123848197632,
9
+ "allocated_bytes_current": 85305508352,
10
+ "allocated_bytes_peak": 123848197632,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 2143632896,
13
+ "inactive_split_bytes_peak": 11517025280,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 131409641472,
21
+ "reserved_bytes_peak": 131409641472,
22
+ "segment_count_current": 1969,
23
+ "segment_count_peak": 1969,
24
+ "timestamp_utc": "2026-08-26T18:53:14.223997Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85305508352,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123848197632,
30
+ "allocated_bytes_peak_delta": 30883538944,
31
+ "before": {
32
+ "active_bytes_current": 85305508352,
33
+ "active_bytes_peak": 92964658688,
34
+ "allocated_bytes_current": 85305508352,
35
+ "allocated_bytes_peak": 92964658688,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 1738882560,
38
+ "inactive_split_bytes_peak": 11517025280,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 103718846464,
46
+ "reserved_bytes_peak": 103718846464,
47
+ "segment_count_current": 1773,
48
+ "segment_count_peak": 1773,
49
+ "timestamp_utc": "2026-08-26T18:53:13.684294Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 2143632896,
55
+ "inactive_split_bytes_current_delta": 404750336,
56
+ "inactive_split_bytes_peak": 11517025280,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 131409641472,
68
+ "reserved_bytes_current_delta": 27690795008,
69
+ "reserved_bytes_peak": 131409641472,
70
+ "reserved_bytes_peak_delta": 27690795008,
71
+ "segment_count_current": 1969,
72
+ "segment_count_current_delta": 196,
73
+ "segment_count_peak": 1969,
74
+ "segment_count_peak_delta": 196,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T185314224423Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:14.273495Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T185314224423Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:618c5c4132390d0279c26422aede52859c7e602d326c07c4f03d0c5c8b884014
3
+ size 2702011
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185314987788Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85305508352,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 125663821824,
5
+ "active_bytes_peak_delta": 1815624192,
6
+ "after": {
7
+ "active_bytes_current": 85305508352,
8
+ "active_bytes_peak": 125663821824,
9
+ "allocated_bytes_current": 85305508352,
10
+ "allocated_bytes_peak": 123851882496,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 2344959488,
13
+ "inactive_split_bytes_peak": 11517025280,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 131409641472,
21
+ "reserved_bytes_peak": 131409641472,
22
+ "segment_count_current": 1969,
23
+ "segment_count_peak": 1969,
24
+ "timestamp_utc": "2026-08-26T18:53:14.987363Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85305508352,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123851882496,
30
+ "allocated_bytes_peak_delta": 3684864,
31
+ "before": {
32
+ "active_bytes_current": 85305508352,
33
+ "active_bytes_peak": 123848197632,
34
+ "allocated_bytes_current": 85305508352,
35
+ "allocated_bytes_peak": 123848197632,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 2143632896,
38
+ "inactive_split_bytes_peak": 11517025280,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 131409641472,
46
+ "reserved_bytes_peak": 131409641472,
47
+ "segment_count_current": 1969,
48
+ "segment_count_peak": 1969,
49
+ "timestamp_utc": "2026-08-26T18:53:14.279853Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 2344959488,
55
+ "inactive_split_bytes_current_delta": 201326592,
56
+ "inactive_split_bytes_peak": 11517025280,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 131409641472,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 131409641472,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1969,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1969,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T185314987788Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:15.035956Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T185314987788Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3612085b1d534624bdcdb31884a77147d8b8f9b10729fd61ccaa8514993da8f4
3
+ size 2662107
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185317851532Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85322551808,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 131086775296,
5
+ "active_bytes_peak_delta": 5422953472,
6
+ "after": {
7
+ "active_bytes_current": 85322551808,
8
+ "active_bytes_peak": 131086775296,
9
+ "allocated_bytes_current": 85322551808,
10
+ "allocated_bytes_peak": 123851882496,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 1747004928,
13
+ "inactive_split_bytes_peak": 12959952384,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 144793665536,
21
+ "reserved_bytes_peak": 144793665536,
22
+ "segment_count_current": 1829,
23
+ "segment_count_peak": 1969,
24
+ "timestamp_utc": "2026-08-26T18:53:17.851076Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85322551808,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 123851882496,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 85305508352,
33
+ "active_bytes_peak": 125663821824,
34
+ "allocated_bytes_current": 85305508352,
35
+ "allocated_bytes_peak": 123851882496,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 2344959488,
38
+ "inactive_split_bytes_peak": 11517025280,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 131409641472,
46
+ "reserved_bytes_peak": 131409641472,
47
+ "segment_count_current": 1969,
48
+ "segment_count_peak": 1969,
49
+ "timestamp_utc": "2026-08-26T18:53:15.042065Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1747004928,
55
+ "inactive_split_bytes_current_delta": -597954560,
56
+ "inactive_split_bytes_peak": 12959952384,
57
+ "inactive_split_bytes_peak_delta": 1442927104,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 144793665536,
68
+ "reserved_bytes_current_delta": 13384024064,
69
+ "reserved_bytes_peak": 144793665536,
70
+ "reserved_bytes_peak_delta": 13384024064,
71
+ "segment_count_current": 1829,
72
+ "segment_count_current_delta": -140,
73
+ "segment_count_peak": 1969,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T185317851532Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:17.905515Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T185317851532Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6cd0d594d5d48552cf0c14b59f08361da894323b9a6a8a39e8e098115e24a667
3
+ size 3557570
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185318232463Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85322551808,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 131086775296,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 85322551808,
8
+ "active_bytes_peak": 131086775296,
9
+ "allocated_bytes_current": 85322551808,
10
+ "allocated_bytes_peak": 123851882496,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 1747004928,
13
+ "inactive_split_bytes_peak": 12959952384,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 144793665536,
21
+ "reserved_bytes_peak": 144793665536,
22
+ "segment_count_current": 1829,
23
+ "segment_count_peak": 1969,
24
+ "timestamp_utc": "2026-08-26T18:53:18.232062Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85322551808,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123851882496,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 85322551808,
33
+ "active_bytes_peak": 131086775296,
34
+ "allocated_bytes_current": 85322551808,
35
+ "allocated_bytes_peak": 123851882496,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 1747004928,
38
+ "inactive_split_bytes_peak": 12959952384,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 144793665536,
46
+ "reserved_bytes_peak": 144793665536,
47
+ "segment_count_current": 1829,
48
+ "segment_count_peak": 1969,
49
+ "timestamp_utc": "2026-08-26T18:53:17.912423Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1747004928,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12959952384,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 144793665536,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 144793665536,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1829,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1969,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T185318232463Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:18.242281Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T185318232463Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d5e8a8bb0df40c64891d9ae6bfa2582ffdbc6a8ae7563522847f1564a50bb1ea
3
+ size 500333
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185318559260Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85322551808,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 131086775296,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 85322551808,
8
+ "active_bytes_peak": 131086775296,
9
+ "allocated_bytes_current": 85322551808,
10
+ "allocated_bytes_peak": 123851882496,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 1747004928,
13
+ "inactive_split_bytes_peak": 12959952384,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 144793665536,
21
+ "reserved_bytes_peak": 144793665536,
22
+ "segment_count_current": 1829,
23
+ "segment_count_peak": 1969,
24
+ "timestamp_utc": "2026-08-26T18:53:18.558867Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85322551808,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123851882496,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 85322551808,
33
+ "active_bytes_peak": 131086775296,
34
+ "allocated_bytes_current": 85322551808,
35
+ "allocated_bytes_peak": 123851882496,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 1747004928,
38
+ "inactive_split_bytes_peak": 12959952384,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 144793665536,
46
+ "reserved_bytes_peak": 144793665536,
47
+ "segment_count_current": 1829,
48
+ "segment_count_peak": 1969,
49
+ "timestamp_utc": "2026-08-26T18:53:18.243412Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1747004928,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12959952384,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 144793665536,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 144793665536,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1829,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1969,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T185318559260Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:18.568683Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T185318559260Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:24e13dc2e7201bffa77a91b8f62cd8b56d6a5fe9d5ddcc5ce109389363fdfdcd
3
+ size 500333
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185313625641Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85295942144,
3
+ "active_bytes_current_delta": 40775332352,
4
+ "active_bytes_peak": 92952749568,
5
+ "active_bytes_peak_delta": 48406973952,
6
+ "after": {
7
+ "active_bytes_current": 85295942144,
8
+ "active_bytes_peak": 92952749568,
9
+ "allocated_bytes_current": 85295942144,
10
+ "allocated_bytes_peak": 92952749568,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 1486304768,
13
+ "inactive_split_bytes_peak": 11855969792,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 103574142976,
21
+ "reserved_bytes_peak": 103574142976,
22
+ "segment_count_current": 1763,
23
+ "segment_count_peak": 1763,
24
+ "timestamp_utc": "2026-08-26T18:53:13.624980Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85295942144,
28
+ "allocated_bytes_current_delta": 40775332352,
29
+ "allocated_bytes_peak": 92952749568,
30
+ "allocated_bytes_peak_delta": 48406973952,
31
+ "before": {
32
+ "active_bytes_current": 44520609792,
33
+ "active_bytes_peak": 44545775616,
34
+ "allocated_bytes_current": 44520609792,
35
+ "allocated_bytes_peak": 44545775616,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 421357568,
38
+ "inactive_split_bytes_peak": 1025337344,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 48251273216,
46
+ "reserved_bytes_peak": 48251273216,
47
+ "segment_count_current": 621,
48
+ "segment_count_peak": 621,
49
+ "timestamp_utc": "2026-08-26T18:52:35.069371Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1486304768,
55
+ "inactive_split_bytes_current_delta": 1064947200,
56
+ "inactive_split_bytes_peak": 11855969792,
57
+ "inactive_split_bytes_peak_delta": 10830632448,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 103574142976,
68
+ "reserved_bytes_current_delta": 55322869760,
69
+ "reserved_bytes_peak": 103574142976,
70
+ "reserved_bytes_peak_delta": 55322869760,
71
+ "segment_count_current": 1763,
72
+ "segment_count_current_delta": 1142,
73
+ "segment_count_peak": 1763,
74
+ "segment_count_peak_delta": 1142,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T185313625641Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:13.676649Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T185313625641Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e646bda37ca523679dba662256b554f942923991e427b8372b97ef4530a348ce
3
+ size 3134744
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185314224386Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85295942144,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 123820125184,
5
+ "active_bytes_peak_delta": 30867375616,
6
+ "after": {
7
+ "active_bytes_current": 85295942144,
8
+ "active_bytes_peak": 123820125184,
9
+ "allocated_bytes_current": 85295942144,
10
+ "allocated_bytes_peak": 123820125184,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 1888957952,
13
+ "inactive_split_bytes_peak": 11855969792,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 131522887680,
21
+ "reserved_bytes_peak": 131522887680,
22
+ "segment_count_current": 1946,
23
+ "segment_count_peak": 1946,
24
+ "timestamp_utc": "2026-08-26T18:53:14.223968Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85295942144,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123820125184,
30
+ "allocated_bytes_peak_delta": 30867375616,
31
+ "before": {
32
+ "active_bytes_current": 85295942144,
33
+ "active_bytes_peak": 92952749568,
34
+ "allocated_bytes_current": 85295942144,
35
+ "allocated_bytes_peak": 92952749568,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 1486304768,
38
+ "inactive_split_bytes_peak": 11855969792,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 103574142976,
46
+ "reserved_bytes_peak": 103574142976,
47
+ "segment_count_current": 1763,
48
+ "segment_count_peak": 1763,
49
+ "timestamp_utc": "2026-08-26T18:53:13.682850Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1888957952,
55
+ "inactive_split_bytes_current_delta": 402653184,
56
+ "inactive_split_bytes_peak": 11855969792,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 131522887680,
68
+ "reserved_bytes_current_delta": 27948744704,
69
+ "reserved_bytes_peak": 131522887680,
70
+ "reserved_bytes_peak_delta": 27948744704,
71
+ "segment_count_current": 1946,
72
+ "segment_count_current_delta": 183,
73
+ "segment_count_peak": 1946,
74
+ "segment_count_peak_delta": 183,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T185314224386Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:14.543073Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T185314224386Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d8f153e6fb4a65e1f916bdba14421432711badae1a069b1a3a02f72c5409007c
3
+ size 2573434
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185314987781Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85295942144,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 125632310272,
5
+ "active_bytes_peak_delta": 1812185088,
6
+ "after": {
7
+ "active_bytes_current": 85295942144,
8
+ "active_bytes_peak": 125632310272,
9
+ "allocated_bytes_current": 85295942144,
10
+ "allocated_bytes_peak": 123820370944,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 2090284544,
13
+ "inactive_split_bytes_peak": 11855969792,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 131522887680,
21
+ "reserved_bytes_peak": 131522887680,
22
+ "segment_count_current": 1946,
23
+ "segment_count_peak": 1946,
24
+ "timestamp_utc": "2026-08-26T18:53:14.987380Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85295942144,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123820370944,
30
+ "allocated_bytes_peak_delta": 245760,
31
+ "before": {
32
+ "active_bytes_current": 85295942144,
33
+ "active_bytes_peak": 123820125184,
34
+ "allocated_bytes_current": 85295942144,
35
+ "allocated_bytes_peak": 123820125184,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 1888957952,
38
+ "inactive_split_bytes_peak": 11855969792,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 131522887680,
46
+ "reserved_bytes_peak": 131522887680,
47
+ "segment_count_current": 1946,
48
+ "segment_count_peak": 1946,
49
+ "timestamp_utc": "2026-08-26T18:53:14.550970Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 2090284544,
55
+ "inactive_split_bytes_current_delta": 201326592,
56
+ "inactive_split_bytes_peak": 11855969792,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 131522887680,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 131522887680,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1946,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1946,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T185314987781Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:15.033243Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T185314987781Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:31636bdd9a3a2ab584a046752635e338b392894ff07510af06acc175c3fe1dbd
3
+ size 2584909
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185317851541Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85312985600,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 131077209088,
5
+ "active_bytes_peak_delta": 5444898816,
6
+ "after": {
7
+ "active_bytes_current": 85312985600,
8
+ "active_bytes_peak": 131077209088,
9
+ "allocated_bytes_current": 85312985600,
10
+ "allocated_bytes_peak": 123829451776,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 1492329984,
13
+ "inactive_split_bytes_peak": 12722853888,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 144529424384,
21
+ "reserved_bytes_peak": 144529424384,
22
+ "segment_count_current": 1800,
23
+ "segment_count_peak": 1946,
24
+ "timestamp_utc": "2026-08-26T18:53:17.851087Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85312985600,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 123829451776,
30
+ "allocated_bytes_peak_delta": 9080832,
31
+ "before": {
32
+ "active_bytes_current": 85295942144,
33
+ "active_bytes_peak": 125632310272,
34
+ "allocated_bytes_current": 85295942144,
35
+ "allocated_bytes_peak": 123820370944,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 2090284544,
38
+ "inactive_split_bytes_peak": 11855969792,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 131522887680,
46
+ "reserved_bytes_peak": 131522887680,
47
+ "segment_count_current": 1946,
48
+ "segment_count_peak": 1946,
49
+ "timestamp_utc": "2026-08-26T18:53:15.038665Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1492329984,
55
+ "inactive_split_bytes_current_delta": -597954560,
56
+ "inactive_split_bytes_peak": 12722853888,
57
+ "inactive_split_bytes_peak_delta": 866884096,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 144529424384,
68
+ "reserved_bytes_current_delta": 13006536704,
69
+ "reserved_bytes_peak": 144529424384,
70
+ "reserved_bytes_peak_delta": 13006536704,
71
+ "segment_count_current": 1800,
72
+ "segment_count_current_delta": -146,
73
+ "segment_count_peak": 1946,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T185317851541Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:17.904663Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T185317851541Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:61489bf2e007af0e6fc131c0df3d99a67bef57dfe023553044ebeb9e86e24783
3
+ size 3223552
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185318232528Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85312985600,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 131077209088,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 85312985600,
8
+ "active_bytes_peak": 131077209088,
9
+ "allocated_bytes_current": 85312985600,
10
+ "allocated_bytes_peak": 123829451776,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 1492329984,
13
+ "inactive_split_bytes_peak": 12722853888,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 144529424384,
21
+ "reserved_bytes_peak": 144529424384,
22
+ "segment_count_current": 1800,
23
+ "segment_count_peak": 1946,
24
+ "timestamp_utc": "2026-08-26T18:53:18.232117Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85312985600,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123829451776,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 85312985600,
33
+ "active_bytes_peak": 131077209088,
34
+ "allocated_bytes_current": 85312985600,
35
+ "allocated_bytes_peak": 123829451776,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 1492329984,
38
+ "inactive_split_bytes_peak": 12722853888,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 144529424384,
46
+ "reserved_bytes_peak": 144529424384,
47
+ "segment_count_current": 1800,
48
+ "segment_count_peak": 1946,
49
+ "timestamp_utc": "2026-08-26T18:53:17.910644Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1492329984,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12722853888,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 144529424384,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 144529424384,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1800,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1946,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T185318232528Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:18.242332Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T185318232528Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:421c3422903e64916555d3dda93bee8254beb070c1a142410b1a4c3f4d41dff7
3
+ size 490746
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185318559265Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85312985600,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 131077209088,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 85312985600,
8
+ "active_bytes_peak": 131077209088,
9
+ "allocated_bytes_current": 85312985600,
10
+ "allocated_bytes_peak": 123829451776,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 1492329984,
13
+ "inactive_split_bytes_peak": 12722853888,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 144529424384,
21
+ "reserved_bytes_peak": 144529424384,
22
+ "segment_count_current": 1800,
23
+ "segment_count_peak": 1946,
24
+ "timestamp_utc": "2026-08-26T18:53:18.558884Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85312985600,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123829451776,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 85312985600,
33
+ "active_bytes_peak": 131077209088,
34
+ "allocated_bytes_current": 85312985600,
35
+ "allocated_bytes_peak": 123829451776,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 1492329984,
38
+ "inactive_split_bytes_peak": 12722853888,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 144529424384,
46
+ "reserved_bytes_peak": 144529424384,
47
+ "segment_count_current": 1800,
48
+ "segment_count_peak": 1946,
49
+ "timestamp_utc": "2026-08-26T18:53:18.243448Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1492329984,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12722853888,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 144529424384,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 144529424384,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1800,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1946,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T185318559265Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:18.569017Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T185318559265Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6839aa3dab10f50ef418ffab11d273249c24a192aca1ebce4e714816aac48090
3
+ size 490746
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185313625630Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85295942144,
3
+ "active_bytes_current_delta": 40775332352,
4
+ "active_bytes_peak": 92952749568,
5
+ "active_bytes_peak_delta": 48406973952,
6
+ "after": {
7
+ "active_bytes_current": 85295942144,
8
+ "active_bytes_peak": 92952749568,
9
+ "allocated_bytes_current": 85295942144,
10
+ "allocated_bytes_peak": 92952749568,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 1486304768,
13
+ "inactive_split_bytes_peak": 11855969792,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 103574142976,
21
+ "reserved_bytes_peak": 103574142976,
22
+ "segment_count_current": 1763,
23
+ "segment_count_peak": 1763,
24
+ "timestamp_utc": "2026-08-26T18:53:13.624970Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85295942144,
28
+ "allocated_bytes_current_delta": 40775332352,
29
+ "allocated_bytes_peak": 92952749568,
30
+ "allocated_bytes_peak_delta": 48406973952,
31
+ "before": {
32
+ "active_bytes_current": 44520609792,
33
+ "active_bytes_peak": 44545775616,
34
+ "allocated_bytes_current": 44520609792,
35
+ "allocated_bytes_peak": 44545775616,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 421357568,
38
+ "inactive_split_bytes_peak": 1025337344,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 48251273216,
46
+ "reserved_bytes_peak": 48251273216,
47
+ "segment_count_current": 621,
48
+ "segment_count_peak": 621,
49
+ "timestamp_utc": "2026-08-26T18:52:35.069260Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1486304768,
55
+ "inactive_split_bytes_current_delta": 1064947200,
56
+ "inactive_split_bytes_peak": 11855969792,
57
+ "inactive_split_bytes_peak_delta": 10830632448,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 103574142976,
68
+ "reserved_bytes_current_delta": 55322869760,
69
+ "reserved_bytes_peak": 103574142976,
70
+ "reserved_bytes_peak_delta": 55322869760,
71
+ "segment_count_current": 1763,
72
+ "segment_count_current_delta": 1142,
73
+ "segment_count_peak": 1763,
74
+ "segment_count_peak_delta": 1142,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T185313625630Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:13.677322Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T185313625630Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:63fa1f05e4028667920d1b5bc237b672f59b4e6e98ddee231347d7614e526bc1
3
+ size 3134747
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185314224359Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85295942144,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 123820125184,
5
+ "active_bytes_peak_delta": 30867375616,
6
+ "after": {
7
+ "active_bytes_current": 85295942144,
8
+ "active_bytes_peak": 123820125184,
9
+ "allocated_bytes_current": 85295942144,
10
+ "allocated_bytes_peak": 123820125184,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 1888957952,
13
+ "inactive_split_bytes_peak": 11855969792,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 131522887680,
21
+ "reserved_bytes_peak": 131522887680,
22
+ "segment_count_current": 1946,
23
+ "segment_count_peak": 1946,
24
+ "timestamp_utc": "2026-08-26T18:53:14.223959Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85295942144,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123820125184,
30
+ "allocated_bytes_peak_delta": 30867375616,
31
+ "before": {
32
+ "active_bytes_current": 85295942144,
33
+ "active_bytes_peak": 92952749568,
34
+ "allocated_bytes_current": 85295942144,
35
+ "allocated_bytes_peak": 92952749568,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 1486304768,
38
+ "inactive_split_bytes_peak": 11855969792,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 103574142976,
46
+ "reserved_bytes_peak": 103574142976,
47
+ "segment_count_current": 1763,
48
+ "segment_count_peak": 1763,
49
+ "timestamp_utc": "2026-08-26T18:53:13.683157Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1888957952,
55
+ "inactive_split_bytes_current_delta": 402653184,
56
+ "inactive_split_bytes_peak": 11855969792,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 131522887680,
68
+ "reserved_bytes_current_delta": 27948744704,
69
+ "reserved_bytes_peak": 131522887680,
70
+ "reserved_bytes_peak_delta": 27948744704,
71
+ "segment_count_current": 1946,
72
+ "segment_count_current_delta": 183,
73
+ "segment_count_peak": 1946,
74
+ "segment_count_peak_delta": 183,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T185314224359Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:14.500901Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T185314224359Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:031c2f06e89858824f1d85cc94f40af6b4435b68367a820ac81ea6dc14b0cef8
3
+ size 2577463
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185314987697Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85295942144,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 125632310272,
5
+ "active_bytes_peak_delta": 1812185088,
6
+ "after": {
7
+ "active_bytes_current": 85295942144,
8
+ "active_bytes_peak": 125632310272,
9
+ "allocated_bytes_current": 85295942144,
10
+ "allocated_bytes_peak": 123820370944,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 2090284544,
13
+ "inactive_split_bytes_peak": 11855969792,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 131522887680,
21
+ "reserved_bytes_peak": 131522887680,
22
+ "segment_count_current": 1946,
23
+ "segment_count_peak": 1946,
24
+ "timestamp_utc": "2026-08-26T18:53:14.987309Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85295942144,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123820370944,
30
+ "allocated_bytes_peak_delta": 245760,
31
+ "before": {
32
+ "active_bytes_current": 85295942144,
33
+ "active_bytes_peak": 123820125184,
34
+ "allocated_bytes_current": 85295942144,
35
+ "allocated_bytes_peak": 123820125184,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 1888957952,
38
+ "inactive_split_bytes_peak": 11855969792,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 131522887680,
46
+ "reserved_bytes_peak": 131522887680,
47
+ "segment_count_current": 1946,
48
+ "segment_count_peak": 1946,
49
+ "timestamp_utc": "2026-08-26T18:53:14.507979Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 2090284544,
55
+ "inactive_split_bytes_current_delta": 201326592,
56
+ "inactive_split_bytes_peak": 11855969792,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 131522887680,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 131522887680,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1946,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1946,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T185314987697Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:15.032068Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T185314987697Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:77c8a786da42b11123c1b507ccba8d75475de0444ac2fb51ab183be836c33169
3
+ size 2585033
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185317851455Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85312985600,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 131077209088,
5
+ "active_bytes_peak_delta": 5444898816,
6
+ "after": {
7
+ "active_bytes_current": 85312985600,
8
+ "active_bytes_peak": 131077209088,
9
+ "allocated_bytes_current": 85312985600,
10
+ "allocated_bytes_peak": 123829451776,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 1492329984,
13
+ "inactive_split_bytes_peak": 12722853888,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 144529424384,
21
+ "reserved_bytes_peak": 144529424384,
22
+ "segment_count_current": 1800,
23
+ "segment_count_peak": 1946,
24
+ "timestamp_utc": "2026-08-26T18:53:17.851038Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85312985600,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 123829451776,
30
+ "allocated_bytes_peak_delta": 9080832,
31
+ "before": {
32
+ "active_bytes_current": 85295942144,
33
+ "active_bytes_peak": 125632310272,
34
+ "allocated_bytes_current": 85295942144,
35
+ "allocated_bytes_peak": 123820370944,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 2090284544,
38
+ "inactive_split_bytes_peak": 11855969792,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 131522887680,
46
+ "reserved_bytes_peak": 131522887680,
47
+ "segment_count_current": 1946,
48
+ "segment_count_peak": 1946,
49
+ "timestamp_utc": "2026-08-26T18:53:15.036850Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1492329984,
55
+ "inactive_split_bytes_current_delta": -597954560,
56
+ "inactive_split_bytes_peak": 12722853888,
57
+ "inactive_split_bytes_peak_delta": 866884096,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 144529424384,
68
+ "reserved_bytes_current_delta": 13006536704,
69
+ "reserved_bytes_peak": 144529424384,
70
+ "reserved_bytes_peak_delta": 13006536704,
71
+ "segment_count_current": 1800,
72
+ "segment_count_current_delta": -146,
73
+ "segment_count_peak": 1946,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T185317851455Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:17.903219Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T185317851455Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9b33652e69f63d473cb99099831160e6df62d171d17ea04635b46203e741ce22
3
+ size 3223709
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185318232431Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85312985600,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 131077209088,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 85312985600,
8
+ "active_bytes_peak": 131077209088,
9
+ "allocated_bytes_current": 85312985600,
10
+ "allocated_bytes_peak": 123829451776,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 1492329984,
13
+ "inactive_split_bytes_peak": 12722853888,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 144529424384,
21
+ "reserved_bytes_peak": 144529424384,
22
+ "segment_count_current": 1800,
23
+ "segment_count_peak": 1946,
24
+ "timestamp_utc": "2026-08-26T18:53:18.232044Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85312985600,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123829451776,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 85312985600,
33
+ "active_bytes_peak": 131077209088,
34
+ "allocated_bytes_current": 85312985600,
35
+ "allocated_bytes_peak": 123829451776,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 1492329984,
38
+ "inactive_split_bytes_peak": 12722853888,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 144529424384,
46
+ "reserved_bytes_peak": 144529424384,
47
+ "segment_count_current": 1800,
48
+ "segment_count_peak": 1946,
49
+ "timestamp_utc": "2026-08-26T18:53:17.908736Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1492329984,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12722853888,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 144529424384,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 144529424384,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1800,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1946,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T185318232431Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:18.241537Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T185318232431Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e1b6f15dd911e1707859a136f7f892cfba0dc62d336b33dfbe5d493d67b45024
3
+ size 490897
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185318559244Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85312985600,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 131077209088,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 85312985600,
8
+ "active_bytes_peak": 131077209088,
9
+ "allocated_bytes_current": 85312985600,
10
+ "allocated_bytes_peak": 123829451776,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 1492329984,
13
+ "inactive_split_bytes_peak": 12722853888,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 144529424384,
21
+ "reserved_bytes_peak": 144529424384,
22
+ "segment_count_current": 1800,
23
+ "segment_count_peak": 1946,
24
+ "timestamp_utc": "2026-08-26T18:53:18.558860Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85312985600,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123829451776,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 85312985600,
33
+ "active_bytes_peak": 131077209088,
34
+ "allocated_bytes_current": 85312985600,
35
+ "allocated_bytes_peak": 123829451776,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 1492329984,
38
+ "inactive_split_bytes_peak": 12722853888,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 144529424384,
46
+ "reserved_bytes_peak": 144529424384,
47
+ "segment_count_current": 1800,
48
+ "segment_count_peak": 1946,
49
+ "timestamp_utc": "2026-08-26T18:53:18.242651Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1492329984,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12722853888,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 144529424384,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 144529424384,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1800,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1946,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T185318559244Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:18.568299Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T185318559244Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d8c34ef05c755ee78cc4804916538de48a6e25f529d16b87cc01c4287c9701f
3
+ size 490897
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185313625648Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85295942144,
3
+ "active_bytes_current_delta": 40775332352,
4
+ "active_bytes_peak": 92952749568,
5
+ "active_bytes_peak_delta": 48406973952,
6
+ "after": {
7
+ "active_bytes_current": 85295942144,
8
+ "active_bytes_peak": 92952749568,
9
+ "allocated_bytes_current": 85295942144,
10
+ "allocated_bytes_peak": 92952749568,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 1507276288,
13
+ "inactive_split_bytes_peak": 11876941312,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 103574142976,
21
+ "reserved_bytes_peak": 103574142976,
22
+ "segment_count_current": 1763,
23
+ "segment_count_peak": 1763,
24
+ "timestamp_utc": "2026-08-26T18:53:13.625012Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85295942144,
28
+ "allocated_bytes_current_delta": 40775332352,
29
+ "allocated_bytes_peak": 92952749568,
30
+ "allocated_bytes_peak_delta": 48406973952,
31
+ "before": {
32
+ "active_bytes_current": 44520609792,
33
+ "active_bytes_peak": 44545775616,
34
+ "allocated_bytes_current": 44520609792,
35
+ "allocated_bytes_peak": 44545775616,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 421357568,
38
+ "inactive_split_bytes_peak": 1025337344,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 48251273216,
46
+ "reserved_bytes_peak": 48251273216,
47
+ "segment_count_current": 621,
48
+ "segment_count_peak": 621,
49
+ "timestamp_utc": "2026-08-26T18:52:35.069348Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1507276288,
55
+ "inactive_split_bytes_current_delta": 1085918720,
56
+ "inactive_split_bytes_peak": 11876941312,
57
+ "inactive_split_bytes_peak_delta": 10851603968,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 103574142976,
68
+ "reserved_bytes_current_delta": 55322869760,
69
+ "reserved_bytes_peak": 103574142976,
70
+ "reserved_bytes_peak_delta": 55322869760,
71
+ "segment_count_current": 1763,
72
+ "segment_count_current_delta": 1142,
73
+ "segment_count_peak": 1763,
74
+ "segment_count_peak_delta": 1142,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T185313625648Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:13.677734Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T185313625648Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:68bddf7f6beb5f3e910a6c5ed9b1ad93a8e578933d3237cae60f7cc12c5ebb39
3
+ size 3134754
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185314224486Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85295942144,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 123820370944,
5
+ "active_bytes_peak_delta": 30867621376,
6
+ "after": {
7
+ "active_bytes_current": 85295942144,
8
+ "active_bytes_peak": 123820370944,
9
+ "allocated_bytes_current": 85295942144,
10
+ "allocated_bytes_peak": 123820370944,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 1909929472,
13
+ "inactive_split_bytes_peak": 11876941312,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 131522887680,
21
+ "reserved_bytes_peak": 131522887680,
22
+ "segment_count_current": 1946,
23
+ "segment_count_peak": 1946,
24
+ "timestamp_utc": "2026-08-26T18:53:14.224057Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85295942144,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123820370944,
30
+ "allocated_bytes_peak_delta": 30867621376,
31
+ "before": {
32
+ "active_bytes_current": 85295942144,
33
+ "active_bytes_peak": 92952749568,
34
+ "allocated_bytes_current": 85295942144,
35
+ "allocated_bytes_peak": 92952749568,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 1507276288,
38
+ "inactive_split_bytes_peak": 11876941312,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 103574142976,
46
+ "reserved_bytes_peak": 103574142976,
47
+ "segment_count_current": 1763,
48
+ "segment_count_peak": 1763,
49
+ "timestamp_utc": "2026-08-26T18:53:13.684182Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1909929472,
55
+ "inactive_split_bytes_current_delta": 402653184,
56
+ "inactive_split_bytes_peak": 11876941312,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 131522887680,
68
+ "reserved_bytes_current_delta": 27948744704,
69
+ "reserved_bytes_peak": 131522887680,
70
+ "reserved_bytes_peak_delta": 27948744704,
71
+ "segment_count_current": 1946,
72
+ "segment_count_current_delta": 183,
73
+ "segment_count_peak": 1946,
74
+ "segment_count_peak_delta": 183,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T185314224486Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:14.575607Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T185314224486Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6d97ef67c9a9ad7a90d17c5de5d8e06d020761c33578e2b3bf83fa036b6b8834
3
+ size 2573452
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185314987802Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85295942144,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 125632556032,
5
+ "active_bytes_peak_delta": 1812185088,
6
+ "after": {
7
+ "active_bytes_current": 85295942144,
8
+ "active_bytes_peak": 125632556032,
9
+ "allocated_bytes_current": 85295942144,
10
+ "allocated_bytes_peak": 123820616704,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 2111256064,
13
+ "inactive_split_bytes_peak": 11876941312,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 131522887680,
21
+ "reserved_bytes_peak": 131522887680,
22
+ "segment_count_current": 1946,
23
+ "segment_count_peak": 1946,
24
+ "timestamp_utc": "2026-08-26T18:53:14.987399Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85295942144,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123820616704,
30
+ "allocated_bytes_peak_delta": 245760,
31
+ "before": {
32
+ "active_bytes_current": 85295942144,
33
+ "active_bytes_peak": 123820370944,
34
+ "allocated_bytes_current": 85295942144,
35
+ "allocated_bytes_peak": 123820370944,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 1909929472,
38
+ "inactive_split_bytes_peak": 11876941312,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 131522887680,
46
+ "reserved_bytes_peak": 131522887680,
47
+ "segment_count_current": 1946,
48
+ "segment_count_peak": 1946,
49
+ "timestamp_utc": "2026-08-26T18:53:14.584007Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 2111256064,
55
+ "inactive_split_bytes_current_delta": 201326592,
56
+ "inactive_split_bytes_peak": 11876941312,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 131522887680,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 131522887680,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1946,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1946,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T185314987802Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:15.035788Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T185314987802Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a88ee4bc057bc19463a395ba10d74e677ea6c279c19f5c5d2f0f288ab9a51376
3
+ size 2584936
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185317851617Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85312985600,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 131077209088,
5
+ "active_bytes_peak_delta": 5444653056,
6
+ "after": {
7
+ "active_bytes_current": 85312985600,
8
+ "active_bytes_peak": 131077209088,
9
+ "allocated_bytes_current": 85312985600,
10
+ "allocated_bytes_peak": 123829451776,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 1513301504,
13
+ "inactive_split_bytes_peak": 12743825408,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 144550395904,
21
+ "reserved_bytes_peak": 144550395904,
22
+ "segment_count_current": 1801,
23
+ "segment_count_peak": 1946,
24
+ "timestamp_utc": "2026-08-26T18:53:17.851160Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85312985600,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 123829451776,
30
+ "allocated_bytes_peak_delta": 8835072,
31
+ "before": {
32
+ "active_bytes_current": 85295942144,
33
+ "active_bytes_peak": 125632556032,
34
+ "allocated_bytes_current": 85295942144,
35
+ "allocated_bytes_peak": 123820616704,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 2111256064,
38
+ "inactive_split_bytes_peak": 11876941312,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 131522887680,
46
+ "reserved_bytes_peak": 131522887680,
47
+ "segment_count_current": 1946,
48
+ "segment_count_peak": 1946,
49
+ "timestamp_utc": "2026-08-26T18:53:15.041759Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1513301504,
55
+ "inactive_split_bytes_current_delta": -597954560,
56
+ "inactive_split_bytes_peak": 12743825408,
57
+ "inactive_split_bytes_peak_delta": 866884096,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 144550395904,
68
+ "reserved_bytes_current_delta": 13027508224,
69
+ "reserved_bytes_peak": 144550395904,
70
+ "reserved_bytes_peak_delta": 13027508224,
71
+ "segment_count_current": 1801,
72
+ "segment_count_current_delta": -145,
73
+ "segment_count_peak": 1946,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T185317851617Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:17.906426Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T185317851617Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2a2f46a2d722b8cf7e9a50687bfc66ebfde665b39e87bf808f629718274ae2da
3
+ size 3223619
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185318232528Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85312985600,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 131077209088,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 85312985600,
8
+ "active_bytes_peak": 131077209088,
9
+ "allocated_bytes_current": 85312985600,
10
+ "allocated_bytes_peak": 123829451776,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 1513301504,
13
+ "inactive_split_bytes_peak": 12743825408,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 144550395904,
21
+ "reserved_bytes_peak": 144550395904,
22
+ "segment_count_current": 1801,
23
+ "segment_count_peak": 1946,
24
+ "timestamp_utc": "2026-08-26T18:53:18.232128Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85312985600,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123829451776,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 85312985600,
33
+ "active_bytes_peak": 131077209088,
34
+ "allocated_bytes_current": 85312985600,
35
+ "allocated_bytes_peak": 123829451776,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 1513301504,
38
+ "inactive_split_bytes_peak": 12743825408,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 144550395904,
46
+ "reserved_bytes_peak": 144550395904,
47
+ "segment_count_current": 1801,
48
+ "segment_count_peak": 1946,
49
+ "timestamp_utc": "2026-08-26T18:53:17.912849Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1513301504,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12743825408,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 144550395904,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 144550395904,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1801,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1946,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T185318232528Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:18.242759Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T185318232528Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ab87d0e2e112218a3040ad8f4b67b6b14dfa6ee8b7e10f608e6b2221e049aca5
3
+ size 490865
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185318559322Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 85312985600,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 131077209088,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 85312985600,
8
+ "active_bytes_peak": 131077209088,
9
+ "allocated_bytes_current": 85312985600,
10
+ "allocated_bytes_peak": 123829451776,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 1513301504,
13
+ "inactive_split_bytes_peak": 12743825408,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 144550395904,
21
+ "reserved_bytes_peak": 144550395904,
22
+ "segment_count_current": 1801,
23
+ "segment_count_peak": 1946,
24
+ "timestamp_utc": "2026-08-26T18:53:18.558939Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 85312985600,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 123829451776,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 85312985600,
33
+ "active_bytes_peak": 131077209088,
34
+ "allocated_bytes_current": 85312985600,
35
+ "allocated_bytes_peak": 123829451776,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 1513301504,
38
+ "inactive_split_bytes_peak": 12743825408,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 144550395904,
46
+ "reserved_bytes_peak": 144550395904,
47
+ "segment_count_current": 1801,
48
+ "segment_count_peak": 1946,
49
+ "timestamp_utc": "2026-08-26T18:53:18.243882Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1513301504,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12743825408,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 144550395904,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 144550395904,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1801,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1946,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T185318559322Z.pickle",
76
+ "timestamp_utc": "2026-08-26T18:53:18.568826Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T185318559322Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:15a5e7858f75d4d9bf75855daff66f2178ac6aefd791d0901817f2b27527f089
3
+ size 490865
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=echo
2
+ config=QWEN3_235B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=better-memory-dump-support
6
+ commit=b7c52442
7
+ num_layers=12
8
+ hidden=4096
9
+ ffn_hidden=12288
10
+ moe_ffn_hidden=1536
11
+ num_experts=128
12
+ topk=8
13
+ experts_per_rank=16
14
+ expert_params=18874368
15
+ heads=64
16
+ kv_groups=4
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=0
20
+ shared_expert_overlap=0
21
+ mla=0
22
+ dense_first_layers=0
23
+ moe_layers=12
24
+ rotary_base=1000000
25
+ rotary_scaling=
26
+ qkv_bias=0
27
+ router_balancing=aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=0
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=65536
39
+ mean_tokens_per_expert=4096
40
+ slot_policy=matched
41
+ slots_per_rank=16
42
+ capacity_mode=static
43
+ capacity=1.25
44
+ cuda_graph=1
45
+ probe_mode=0
46
+ async_grad_reduce=0
47
+ branch_role=memory
48
+ memory_dump_mode=iteration
49
+ memory_snapshot_path=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=6
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
echo/tierbmem/tierbmem_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff