AbhishekGhosh1998 commited on
Commit
8d25378
·
verified ·
1 Parent(s): 468c662

granularity results echo/tierbmem @ 2026-08-26T19:10:38Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 (part 3)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190754565447Z.pickle +3 -0
  2. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190756149977Z.json +79 -0
  3. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190756149977Z.pickle +3 -0
  4. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190756378100Z.json +79 -0
  5. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190756378100Z.pickle +3 -0
  6. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190756568179Z.json +79 -0
  7. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190756568179Z.pickle +3 -0
  8. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190753657100Z.json +79 -0
  9. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190753657100Z.pickle +3 -0
  10. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190754288396Z.json +79 -0
  11. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190754288396Z.pickle +3 -0
  12. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190754565483Z.json +79 -0
  13. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190754565483Z.pickle +3 -0
  14. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190756150033Z.json +79 -0
  15. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190756150033Z.pickle +3 -0
  16. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190756378085Z.json +79 -0
  17. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190756378085Z.pickle +3 -0
  18. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190756568263Z.json +79 -0
  19. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190756568263Z.pickle +3 -0
  20. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190753657099Z.json +79 -0
  21. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190753657099Z.pickle +3 -0
  22. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190754288400Z.json +79 -0
  23. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190754288400Z.pickle +3 -0
  24. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190754565475Z.json +79 -0
  25. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190754565475Z.pickle +3 -0
  26. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190756149994Z.json +79 -0
  27. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190756149994Z.pickle +3 -0
  28. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190756378117Z.json +79 -0
  29. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190756378117Z.pickle +3 -0
  30. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190756568206Z.json +79 -0
  31. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190756568206Z.pickle +3 -0
  32. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190753657285Z.json +79 -0
  33. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190753657285Z.pickle +3 -0
  34. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190754288383Z.json +79 -0
  35. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190754288383Z.pickle +3 -0
  36. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190754565485Z.json +79 -0
  37. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190754565485Z.pickle +3 -0
  38. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190756149994Z.json +79 -0
  39. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190756149994Z.pickle +3 -0
  40. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190756378065Z.json +79 -0
  41. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190756378065Z.pickle +3 -0
  42. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190756568213Z.json +79 -0
  43. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190756568213Z.pickle +3 -0
  44. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190753657101Z.json +79 -0
  45. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190753657101Z.pickle +3 -0
  46. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190754288388Z.json +79 -0
  47. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190754288388Z.pickle +3 -0
  48. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190754566103Z.json +79 -0
  49. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190754566103Z.pickle +3 -0
  50. echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190756150024Z.json +79 -0
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000002_20260826T190754565447Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9d24c9da0aac98760181ec8f6f17923ffdf043960281198b845b188095a05a0a
3
+ size 1706466
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190756149977Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72085230592,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 99464647168,
5
+ "active_bytes_peak_delta": 4390016000,
6
+ "after": {
7
+ "active_bytes_current": 72085230592,
8
+ "active_bytes_peak": 99464647168,
9
+ "allocated_bytes_current": 72085230592,
10
+ "allocated_bytes_peak": 95074631168,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 1931652096,
13
+ "inactive_split_bytes_peak": 5669243904,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 108192071680,
21
+ "reserved_bytes_peak": 108192071680,
22
+ "segment_count_current": 1166,
23
+ "segment_count_peak": 1166,
24
+ "timestamp_utc": "2026-08-26T19:07:56.149539Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72085230592,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 95074631168,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 72068187136,
33
+ "active_bytes_peak": 95074631168,
34
+ "allocated_bytes_current": 72068187136,
35
+ "allocated_bytes_peak": 95074631168,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 2028387328,
38
+ "inactive_split_bytes_peak": 4951808512,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 100717821952,
46
+ "reserved_bytes_peak": 100717821952,
47
+ "segment_count_current": 1144,
48
+ "segment_count_peak": 1144,
49
+ "timestamp_utc": "2026-08-26T19:07:54.597941Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1931652096,
55
+ "inactive_split_bytes_current_delta": -96735232,
56
+ "inactive_split_bytes_peak": 5669243904,
57
+ "inactive_split_bytes_peak_delta": 717435392,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 108192071680,
68
+ "reserved_bytes_current_delta": 7474249728,
69
+ "reserved_bytes_peak": 108192071680,
70
+ "reserved_bytes_peak_delta": 7474249728,
71
+ "segment_count_current": 1166,
72
+ "segment_count_current_delta": 22,
73
+ "segment_count_peak": 1166,
74
+ "segment_count_peak_delta": 22,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260826T190756149977Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:56.185975Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000003_20260826T190756149977Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c8aed3b8a6cbd2f1450357cf223e62221bd6bbf5dc29d0ed5316ccec631d2a37
3
+ size 2102583
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190756378100Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72085230592,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 99464647168,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 72085230592,
8
+ "active_bytes_peak": 99464647168,
9
+ "allocated_bytes_current": 72085230592,
10
+ "allocated_bytes_peak": 95074631168,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 1931652096,
13
+ "inactive_split_bytes_peak": 5669243904,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 108192071680,
21
+ "reserved_bytes_peak": 108192071680,
22
+ "segment_count_current": 1166,
23
+ "segment_count_peak": 1166,
24
+ "timestamp_utc": "2026-08-26T19:07:56.377676Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72085230592,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 95074631168,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 72085230592,
33
+ "active_bytes_peak": 99464647168,
34
+ "allocated_bytes_current": 72085230592,
35
+ "allocated_bytes_peak": 95074631168,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 1931652096,
38
+ "inactive_split_bytes_peak": 5669243904,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 108192071680,
46
+ "reserved_bytes_peak": 108192071680,
47
+ "segment_count_current": 1166,
48
+ "segment_count_peak": 1166,
49
+ "timestamp_utc": "2026-08-26T19:07:56.190648Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1931652096,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 5669243904,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 108192071680,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 108192071680,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1166,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1166,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260826T190756378100Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:56.385355Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000004_20260826T190756378100Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b05a572663b534cc1983bb0494ec9d597f0839b445405b6b9594fe4cdde71f30
3
+ size 331653
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190756568179Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72085230592,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 99464647168,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 72085230592,
8
+ "active_bytes_peak": 99464647168,
9
+ "allocated_bytes_current": 72085230592,
10
+ "allocated_bytes_peak": 95074631168,
11
+ "device": 1,
12
+ "inactive_split_bytes_current": 1931652096,
13
+ "inactive_split_bytes_peak": 5669243904,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 1,
20
+ "reserved_bytes_current": 108192071680,
21
+ "reserved_bytes_peak": 108192071680,
22
+ "segment_count_current": 1166,
23
+ "segment_count_peak": 1166,
24
+ "timestamp_utc": "2026-08-26T19:07:56.567795Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72085230592,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 95074631168,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 72085230592,
33
+ "active_bytes_peak": 99464647168,
34
+ "allocated_bytes_current": 72085230592,
35
+ "allocated_bytes_peak": 95074631168,
36
+ "device": 1,
37
+ "inactive_split_bytes_current": 1931652096,
38
+ "inactive_split_bytes_peak": 5669243904,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 1,
45
+ "reserved_bytes_current": 108192071680,
46
+ "reserved_bytes_peak": 108192071680,
47
+ "segment_count_current": 1166,
48
+ "segment_count_peak": 1166,
49
+ "timestamp_utc": "2026-08-26T19:07:56.386363Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 1,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1931652096,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 5669243904,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 1,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 108192071680,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 108192071680,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1166,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1166,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260826T190756568179Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:56.575238Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0001_iter000005_20260826T190756568179Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:caf72eb6a4500079fb7a27beb13c68170b12803ce2b1ee37ffca8f8dc3d46af3
3
+ size 331653
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190753657100Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72067058688,
3
+ "active_bytes_current_delta": 31381090304,
4
+ "active_bytes_peak": 74050460672,
5
+ "active_bytes_peak_delta": 33326743552,
6
+ "after": {
7
+ "active_bytes_current": 72067058688,
8
+ "active_bytes_peak": 74050460672,
9
+ "allocated_bytes_current": 72067058688,
10
+ "allocated_bytes_peak": 74050460672,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 1679291392,
13
+ "inactive_split_bytes_peak": 4932083200,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 84494254080,
21
+ "reserved_bytes_peak": 84494254080,
22
+ "segment_count_current": 1046,
23
+ "segment_count_peak": 1046,
24
+ "timestamp_utc": "2026-08-26T19:07:53.656629Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72067058688,
28
+ "allocated_bytes_current_delta": 31381090304,
29
+ "allocated_bytes_peak": 74050460672,
30
+ "allocated_bytes_peak_delta": 33326743552,
31
+ "before": {
32
+ "active_bytes_current": 40685968384,
33
+ "active_bytes_peak": 40723717120,
34
+ "allocated_bytes_current": 40685968384,
35
+ "allocated_bytes_peak": 40723717120,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 283993088,
38
+ "inactive_split_bytes_peak": 504194048,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 43130028032,
46
+ "reserved_bytes_peak": 43130028032,
47
+ "segment_count_current": 251,
48
+ "segment_count_peak": 251,
49
+ "timestamp_utc": "2026-08-26T19:07:14.578912Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1679291392,
55
+ "inactive_split_bytes_current_delta": 1395298304,
56
+ "inactive_split_bytes_peak": 4932083200,
57
+ "inactive_split_bytes_peak_delta": 4427889152,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 84494254080,
68
+ "reserved_bytes_current_delta": 41364226048,
69
+ "reserved_bytes_peak": 84494254080,
70
+ "reserved_bytes_peak_delta": 41364226048,
71
+ "segment_count_current": 1046,
72
+ "segment_count_current_delta": 795,
73
+ "segment_count_peak": 1046,
74
+ "segment_count_peak_delta": 795,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260826T190753657100Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:53.692854Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000000_20260826T190753657100Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9974cd72c5633b571d5ce6856e67df6dafcda06ff0059ce8a11a706bc3fa2877
3
+ size 1796624
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190754288396Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72067058688,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 95073281024,
5
+ "active_bytes_peak_delta": 21022820352,
6
+ "after": {
7
+ "active_bytes_current": 72067058688,
8
+ "active_bytes_peak": 95073281024,
9
+ "allocated_bytes_current": 72067058688,
10
+ "allocated_bytes_peak": 95073281024,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 1974989824,
13
+ "inactive_split_bytes_peak": 4932083200,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 100428414976,
21
+ "reserved_bytes_peak": 100428414976,
22
+ "segment_count_current": 1145,
23
+ "segment_count_peak": 1145,
24
+ "timestamp_utc": "2026-08-26T19:07:54.287965Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72067058688,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 95073281024,
30
+ "allocated_bytes_peak_delta": 21022820352,
31
+ "before": {
32
+ "active_bytes_current": 72067058688,
33
+ "active_bytes_peak": 74050460672,
34
+ "allocated_bytes_current": 72067058688,
35
+ "allocated_bytes_peak": 74050460672,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 1679291392,
38
+ "inactive_split_bytes_peak": 4932083200,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 84494254080,
46
+ "reserved_bytes_peak": 84494254080,
47
+ "segment_count_current": 1046,
48
+ "segment_count_peak": 1046,
49
+ "timestamp_utc": "2026-08-26T19:07:53.697327Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1974989824,
55
+ "inactive_split_bytes_current_delta": 295698432,
56
+ "inactive_split_bytes_peak": 4932083200,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 100428414976,
68
+ "reserved_bytes_current_delta": 15934160896,
69
+ "reserved_bytes_peak": 100428414976,
70
+ "reserved_bytes_peak_delta": 15934160896,
71
+ "segment_count_current": 1145,
72
+ "segment_count_current_delta": 99,
73
+ "segment_count_peak": 1145,
74
+ "segment_count_peak_delta": 99,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260826T190754288396Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:54.320452Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000001_20260826T190754288396Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6e1342c84c3066cf8a9e8a290ce04b4218ac538ebcc5633c1372fef50bc9d88f
3
+ size 1695924
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190754565483Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72067058688,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 95073281024,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 72067058688,
8
+ "active_bytes_peak": 95073281024,
9
+ "allocated_bytes_current": 72067058688,
10
+ "allocated_bytes_peak": 95073281024,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 1974989824,
13
+ "inactive_split_bytes_peak": 4932083200,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 101252595712,
21
+ "reserved_bytes_peak": 101252595712,
22
+ "segment_count_current": 1146,
23
+ "segment_count_peak": 1146,
24
+ "timestamp_utc": "2026-08-26T19:07:54.565080Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72067058688,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 95073281024,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 72067058688,
33
+ "active_bytes_peak": 95073281024,
34
+ "allocated_bytes_current": 72067058688,
35
+ "allocated_bytes_peak": 95073281024,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 1974989824,
38
+ "inactive_split_bytes_peak": 4932083200,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 100428414976,
46
+ "reserved_bytes_peak": 100428414976,
47
+ "segment_count_current": 1145,
48
+ "segment_count_peak": 1145,
49
+ "timestamp_utc": "2026-08-26T19:07:54.324314Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1974989824,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 4932083200,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 101252595712,
68
+ "reserved_bytes_current_delta": 824180736,
69
+ "reserved_bytes_peak": 101252595712,
70
+ "reserved_bytes_peak_delta": 824180736,
71
+ "segment_count_current": 1146,
72
+ "segment_count_current_delta": 1,
73
+ "segment_count_peak": 1146,
74
+ "segment_count_peak_delta": 1,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260826T190754565483Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:54.593215Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000002_20260826T190754565483Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0fbb0d9fbd45546f2a956bbe610efb15010ac85ccea61be642a9307391993f15
3
+ size 1709822
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190756150033Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72084102144,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 99463518720,
5
+ "active_bytes_peak_delta": 4390237696,
6
+ "after": {
7
+ "active_bytes_current": 72084102144,
8
+ "active_bytes_peak": 99463518720,
9
+ "allocated_bytes_current": 72084102144,
10
+ "allocated_bytes_peak": 95073281024,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 1874060288,
13
+ "inactive_split_bytes_peak": 5609554944,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 108137545728,
21
+ "reserved_bytes_peak": 108137545728,
22
+ "segment_count_current": 1166,
23
+ "segment_count_peak": 1166,
24
+ "timestamp_utc": "2026-08-26T19:07:56.149582Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72084102144,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 95073281024,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 72067058688,
33
+ "active_bytes_peak": 95073281024,
34
+ "allocated_bytes_current": 72067058688,
35
+ "allocated_bytes_peak": 95073281024,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 1974989824,
38
+ "inactive_split_bytes_peak": 4932083200,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 101252595712,
46
+ "reserved_bytes_peak": 101252595712,
47
+ "segment_count_current": 1146,
48
+ "segment_count_peak": 1146,
49
+ "timestamp_utc": "2026-08-26T19:07:54.596691Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1874060288,
55
+ "inactive_split_bytes_current_delta": -100929536,
56
+ "inactive_split_bytes_peak": 5609554944,
57
+ "inactive_split_bytes_peak_delta": 677471744,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 108137545728,
68
+ "reserved_bytes_current_delta": 6884950016,
69
+ "reserved_bytes_peak": 108137545728,
70
+ "reserved_bytes_peak_delta": 6884950016,
71
+ "segment_count_current": 1166,
72
+ "segment_count_current_delta": 20,
73
+ "segment_count_peak": 1166,
74
+ "segment_count_peak_delta": 20,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260826T190756150033Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:56.186358Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000003_20260826T190756150033Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f07126cc4e9526702c515dfd2355fbcc4ce529a364dc88dd48932a58c562dcc8
3
+ size 1933754
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190756378085Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72084102144,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 99463518720,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 72084102144,
8
+ "active_bytes_peak": 99463518720,
9
+ "allocated_bytes_current": 72084102144,
10
+ "allocated_bytes_peak": 95073281024,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 1874060288,
13
+ "inactive_split_bytes_peak": 5609554944,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 108137545728,
21
+ "reserved_bytes_peak": 108137545728,
22
+ "segment_count_current": 1166,
23
+ "segment_count_peak": 1166,
24
+ "timestamp_utc": "2026-08-26T19:07:56.377683Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72084102144,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 95073281024,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 72084102144,
33
+ "active_bytes_peak": 99463518720,
34
+ "allocated_bytes_current": 72084102144,
35
+ "allocated_bytes_peak": 95073281024,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 1874060288,
38
+ "inactive_split_bytes_peak": 5609554944,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 108137545728,
46
+ "reserved_bytes_peak": 108137545728,
47
+ "segment_count_current": 1166,
48
+ "segment_count_peak": 1166,
49
+ "timestamp_utc": "2026-08-26T19:07:56.190467Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1874060288,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 5609554944,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 108137545728,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 108137545728,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1166,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1166,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260826T190756378085Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:56.385340Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000004_20260826T190756378085Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d83f263d9ad0ad4dd8dcf3ad71c2ed2b4e16c45d8d739f8e02fdacd3110084b1
3
+ size 307966
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190756568263Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72084102144,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 99463518720,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 72084102144,
8
+ "active_bytes_peak": 99463518720,
9
+ "allocated_bytes_current": 72084102144,
10
+ "allocated_bytes_peak": 95073281024,
11
+ "device": 2,
12
+ "inactive_split_bytes_current": 1874060288,
13
+ "inactive_split_bytes_peak": 5609554944,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 2,
20
+ "reserved_bytes_current": 108137545728,
21
+ "reserved_bytes_peak": 108137545728,
22
+ "segment_count_current": 1166,
23
+ "segment_count_peak": 1166,
24
+ "timestamp_utc": "2026-08-26T19:07:56.567869Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72084102144,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 95073281024,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 72084102144,
33
+ "active_bytes_peak": 99463518720,
34
+ "allocated_bytes_current": 72084102144,
35
+ "allocated_bytes_peak": 95073281024,
36
+ "device": 2,
37
+ "inactive_split_bytes_current": 1874060288,
38
+ "inactive_split_bytes_peak": 5609554944,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 2,
45
+ "reserved_bytes_current": 108137545728,
46
+ "reserved_bytes_peak": 108137545728,
47
+ "segment_count_current": 1166,
48
+ "segment_count_peak": 1166,
49
+ "timestamp_utc": "2026-08-26T19:07:56.386351Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 2,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1874060288,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 5609554944,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 2,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 108137545728,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 108137545728,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1166,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1166,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260826T190756568263Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:56.575890Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0002_iter000005_20260826T190756568263Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1c9bd3551ce330ca7aa01ba1c3f9ba3a2793be0da75f2393ed0483f90232bcaf
3
+ size 307935
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190753657099Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72076615168,
3
+ "active_bytes_current_delta": 31390646784,
4
+ "active_bytes_peak": 74041179136,
5
+ "active_bytes_peak_delta": 33317462016,
6
+ "after": {
7
+ "active_bytes_current": 72076615168,
8
+ "active_bytes_peak": 74041179136,
9
+ "allocated_bytes_current": 72076615168,
10
+ "allocated_bytes_peak": 74041179136,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 1835409920,
13
+ "inactive_split_bytes_peak": 5056163328,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 85135982592,
21
+ "reserved_bytes_peak": 85135982592,
22
+ "segment_count_current": 1021,
23
+ "segment_count_peak": 1021,
24
+ "timestamp_utc": "2026-08-26T19:07:53.656598Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72076615168,
28
+ "allocated_bytes_current_delta": 31390646784,
29
+ "allocated_bytes_peak": 74041179136,
30
+ "allocated_bytes_peak_delta": 33317462016,
31
+ "before": {
32
+ "active_bytes_current": 40685968384,
33
+ "active_bytes_peak": 40723717120,
34
+ "allocated_bytes_current": 40685968384,
35
+ "allocated_bytes_peak": 40723717120,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 508388352,
38
+ "inactive_split_bytes_peak": 728589312,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 43864031232,
46
+ "reserved_bytes_peak": 43864031232,
47
+ "segment_count_current": 261,
48
+ "segment_count_peak": 261,
49
+ "timestamp_utc": "2026-08-26T19:07:14.578843Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1835409920,
55
+ "inactive_split_bytes_current_delta": 1327021568,
56
+ "inactive_split_bytes_peak": 5056163328,
57
+ "inactive_split_bytes_peak_delta": 4327574016,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 85135982592,
68
+ "reserved_bytes_current_delta": 41271951360,
69
+ "reserved_bytes_peak": 85135982592,
70
+ "reserved_bytes_peak_delta": 41271951360,
71
+ "segment_count_current": 1021,
72
+ "segment_count_current_delta": 760,
73
+ "segment_count_peak": 1021,
74
+ "segment_count_peak_delta": 760,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260826T190753657099Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:53.693208Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000000_20260826T190753657099Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b3a60d377640686a765e132ac35e04aa64458b689a830bafe90b8657ffdb7cf1
3
+ size 2050482
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190754288400Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72076615168,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 95064244224,
5
+ "active_bytes_peak_delta": 21023065088,
6
+ "after": {
7
+ "active_bytes_current": 72076615168,
8
+ "active_bytes_peak": 95064244224,
9
+ "allocated_bytes_current": 72076615168,
10
+ "allocated_bytes_peak": 95064244224,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 2131108352,
13
+ "inactive_split_bytes_peak": 5056163328,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 100415832064,
21
+ "reserved_bytes_peak": 100415832064,
22
+ "segment_count_current": 1115,
23
+ "segment_count_peak": 1115,
24
+ "timestamp_utc": "2026-08-26T19:07:54.287978Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72076615168,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 95064244224,
30
+ "allocated_bytes_peak_delta": 21023065088,
31
+ "before": {
32
+ "active_bytes_current": 72076615168,
33
+ "active_bytes_peak": 74041179136,
34
+ "allocated_bytes_current": 72076615168,
35
+ "allocated_bytes_peak": 74041179136,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 1835409920,
38
+ "inactive_split_bytes_peak": 5056163328,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 85135982592,
46
+ "reserved_bytes_peak": 85135982592,
47
+ "segment_count_current": 1021,
48
+ "segment_count_peak": 1021,
49
+ "timestamp_utc": "2026-08-26T19:07:53.697502Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 2131108352,
55
+ "inactive_split_bytes_current_delta": 295698432,
56
+ "inactive_split_bytes_peak": 5056163328,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 100415832064,
68
+ "reserved_bytes_current_delta": 15279849472,
69
+ "reserved_bytes_peak": 100415832064,
70
+ "reserved_bytes_peak_delta": 15279849472,
71
+ "segment_count_current": 1115,
72
+ "segment_count_current_delta": 94,
73
+ "segment_count_peak": 1115,
74
+ "segment_count_peak_delta": 94,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260826T190754288400Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:54.320860Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000001_20260826T190754288400Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3a19a0a93d892ee5ad86e5c3730c7d4bde91f9d647376d6d844828a6589a47f3
3
+ size 1669210
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190754565475Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72076615168,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 95064244224,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 72076615168,
8
+ "active_bytes_peak": 95064244224,
9
+ "allocated_bytes_current": 72076615168,
10
+ "allocated_bytes_peak": 95064244224,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 2131108352,
13
+ "inactive_split_bytes_peak": 5056163328,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 100415832064,
21
+ "reserved_bytes_peak": 100415832064,
22
+ "segment_count_current": 1115,
23
+ "segment_count_peak": 1115,
24
+ "timestamp_utc": "2026-08-26T19:07:54.565074Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72076615168,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 95064244224,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 72076615168,
33
+ "active_bytes_peak": 95064244224,
34
+ "allocated_bytes_current": 72076615168,
35
+ "allocated_bytes_peak": 95064244224,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 2131108352,
38
+ "inactive_split_bytes_peak": 5056163328,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 100415832064,
46
+ "reserved_bytes_peak": 100415832064,
47
+ "segment_count_current": 1115,
48
+ "segment_count_peak": 1115,
49
+ "timestamp_utc": "2026-08-26T19:07:54.324647Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 2131108352,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 5056163328,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 100415832064,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 100415832064,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1115,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1115,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260826T190754565475Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:54.593327Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000002_20260826T190754565475Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fe92993356aa1c104dce5ece68955e5b920b360d36efcc461ffb5424a5ef8b9b
3
+ size 1659945
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190756149994Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72093658624,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 99473075200,
5
+ "active_bytes_peak_delta": 4408830976,
6
+ "after": {
7
+ "active_bytes_current": 72093658624,
8
+ "active_bytes_peak": 99473075200,
9
+ "allocated_bytes_current": 72093658624,
10
+ "allocated_bytes_peak": 95069056000,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 2030178816,
13
+ "inactive_split_bytes_peak": 5803422208,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 108303220736,
21
+ "reserved_bytes_peak": 108303220736,
22
+ "segment_count_current": 1158,
23
+ "segment_count_peak": 1158,
24
+ "timestamp_utc": "2026-08-26T19:07:56.149547Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72093658624,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 95069056000,
30
+ "allocated_bytes_peak_delta": 4811776,
31
+ "before": {
32
+ "active_bytes_current": 72076615168,
33
+ "active_bytes_peak": 95064244224,
34
+ "allocated_bytes_current": 72076615168,
35
+ "allocated_bytes_peak": 95064244224,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 2131108352,
38
+ "inactive_split_bytes_peak": 5056163328,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 100415832064,
46
+ "reserved_bytes_peak": 100415832064,
47
+ "segment_count_current": 1115,
48
+ "segment_count_peak": 1115,
49
+ "timestamp_utc": "2026-08-26T19:07:54.596724Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 2030178816,
55
+ "inactive_split_bytes_current_delta": -100929536,
56
+ "inactive_split_bytes_peak": 5803422208,
57
+ "inactive_split_bytes_peak_delta": 747258880,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 108303220736,
68
+ "reserved_bytes_current_delta": 7887388672,
69
+ "reserved_bytes_peak": 108303220736,
70
+ "reserved_bytes_peak_delta": 7887388672,
71
+ "segment_count_current": 1158,
72
+ "segment_count_current_delta": 43,
73
+ "segment_count_peak": 1158,
74
+ "segment_count_peak_delta": 43,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260826T190756149994Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:56.184118Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000003_20260826T190756149994Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:13f830338122f0b64b45e22a5f1604c02d4f492f2b439322fa1273892a31c950
3
+ size 1929358
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190756378117Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72093658624,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 99473075200,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 72093658624,
8
+ "active_bytes_peak": 99473075200,
9
+ "allocated_bytes_current": 72093658624,
10
+ "allocated_bytes_peak": 95069056000,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 2030178816,
13
+ "inactive_split_bytes_peak": 5803422208,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 108303220736,
21
+ "reserved_bytes_peak": 108303220736,
22
+ "segment_count_current": 1158,
23
+ "segment_count_peak": 1158,
24
+ "timestamp_utc": "2026-08-26T19:07:56.377701Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72093658624,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 95069056000,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 72093658624,
33
+ "active_bytes_peak": 99473075200,
34
+ "allocated_bytes_current": 72093658624,
35
+ "allocated_bytes_peak": 95069056000,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 2030178816,
38
+ "inactive_split_bytes_peak": 5803422208,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 108303220736,
46
+ "reserved_bytes_peak": 108303220736,
47
+ "segment_count_current": 1158,
48
+ "segment_count_peak": 1158,
49
+ "timestamp_utc": "2026-08-26T19:07:56.188145Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 2030178816,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 5803422208,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 108303220736,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 108303220736,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1158,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1158,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260826T190756378117Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:56.385243Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000004_20260826T190756378117Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:30bb8b8c390915527946035eb877a81d8e81b0ca011a02f951384fd09a6d3623
3
+ size 304763
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190756568206Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72093658624,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 99473075200,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 72093658624,
8
+ "active_bytes_peak": 99473075200,
9
+ "allocated_bytes_current": 72093658624,
10
+ "allocated_bytes_peak": 95069056000,
11
+ "device": 3,
12
+ "inactive_split_bytes_current": 2030178816,
13
+ "inactive_split_bytes_peak": 5803422208,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 3,
20
+ "reserved_bytes_current": 108303220736,
21
+ "reserved_bytes_peak": 108303220736,
22
+ "segment_count_current": 1158,
23
+ "segment_count_peak": 1158,
24
+ "timestamp_utc": "2026-08-26T19:07:56.567824Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72093658624,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 95069056000,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 72093658624,
33
+ "active_bytes_peak": 99473075200,
34
+ "allocated_bytes_current": 72093658624,
35
+ "allocated_bytes_peak": 95069056000,
36
+ "device": 3,
37
+ "inactive_split_bytes_current": 2030178816,
38
+ "inactive_split_bytes_peak": 5803422208,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 3,
45
+ "reserved_bytes_current": 108303220736,
46
+ "reserved_bytes_peak": 108303220736,
47
+ "segment_count_current": 1158,
48
+ "segment_count_peak": 1158,
49
+ "timestamp_utc": "2026-08-26T19:07:56.386233Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 3,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 2030178816,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 5803422208,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 3,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 108303220736,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 108303220736,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1158,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1158,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260826T190756568206Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:56.575037Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0003_iter000005_20260826T190756568206Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:442e464633ddb74b52d832884d0d5808bfd5f065f0c596eb24804e6d095d9442
3
+ size 304732
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190753657285Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72076615168,
3
+ "active_bytes_current_delta": 31390646784,
4
+ "active_bytes_peak": 74041179136,
5
+ "active_bytes_peak_delta": 33317462016,
6
+ "after": {
7
+ "active_bytes_current": 72076615168,
8
+ "active_bytes_peak": 74041179136,
9
+ "allocated_bytes_current": 72076615168,
10
+ "allocated_bytes_peak": 74041179136,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 1835409920,
13
+ "inactive_split_bytes_peak": 5056163328,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 84255178752,
21
+ "reserved_bytes_peak": 84255178752,
22
+ "segment_count_current": 1018,
23
+ "segment_count_peak": 1018,
24
+ "timestamp_utc": "2026-08-26T19:07:53.656779Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72076615168,
28
+ "allocated_bytes_current_delta": 31390646784,
29
+ "allocated_bytes_peak": 74041179136,
30
+ "allocated_bytes_peak_delta": 33317462016,
31
+ "before": {
32
+ "active_bytes_current": 40685968384,
33
+ "active_bytes_peak": 40723717120,
34
+ "allocated_bytes_current": 40685968384,
35
+ "allocated_bytes_peak": 40723717120,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 508388352,
38
+ "inactive_split_bytes_peak": 728589312,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 43864031232,
46
+ "reserved_bytes_peak": 43864031232,
47
+ "segment_count_current": 261,
48
+ "segment_count_peak": 261,
49
+ "timestamp_utc": "2026-08-26T19:07:14.578764Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1835409920,
55
+ "inactive_split_bytes_current_delta": 1327021568,
56
+ "inactive_split_bytes_peak": 5056163328,
57
+ "inactive_split_bytes_peak_delta": 4327574016,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 84255178752,
68
+ "reserved_bytes_current_delta": 40391147520,
69
+ "reserved_bytes_peak": 84255178752,
70
+ "reserved_bytes_peak_delta": 40391147520,
71
+ "segment_count_current": 1018,
72
+ "segment_count_current_delta": 757,
73
+ "segment_count_peak": 1018,
74
+ "segment_count_peak_delta": 757,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T190753657285Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:53.694479Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190753657285Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:40dc7eed517b5ebfbe4f9051bfe06ee8e4753f811601822a4a2226b0a87057d2
3
+ size 2049975
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190754288383Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72076615168,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 95064244224,
5
+ "active_bytes_peak_delta": 21023065088,
6
+ "after": {
7
+ "active_bytes_current": 72076615168,
8
+ "active_bytes_peak": 95064244224,
9
+ "allocated_bytes_current": 72076615168,
10
+ "allocated_bytes_peak": 95064244224,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 2131108352,
13
+ "inactive_split_bytes_peak": 5056163328,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 100709433344,
21
+ "reserved_bytes_peak": 100709433344,
22
+ "segment_count_current": 1116,
23
+ "segment_count_peak": 1116,
24
+ "timestamp_utc": "2026-08-26T19:07:54.287961Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72076615168,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 95064244224,
30
+ "allocated_bytes_peak_delta": 21023065088,
31
+ "before": {
32
+ "active_bytes_current": 72076615168,
33
+ "active_bytes_peak": 74041179136,
34
+ "allocated_bytes_current": 72076615168,
35
+ "allocated_bytes_peak": 74041179136,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 1835409920,
38
+ "inactive_split_bytes_peak": 5056163328,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 84255178752,
46
+ "reserved_bytes_peak": 84255178752,
47
+ "segment_count_current": 1018,
48
+ "segment_count_peak": 1018,
49
+ "timestamp_utc": "2026-08-26T19:07:53.698779Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 2131108352,
55
+ "inactive_split_bytes_current_delta": 295698432,
56
+ "inactive_split_bytes_peak": 5056163328,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 100709433344,
68
+ "reserved_bytes_current_delta": 16454254592,
69
+ "reserved_bytes_peak": 100709433344,
70
+ "reserved_bytes_peak_delta": 16454254592,
71
+ "segment_count_current": 1116,
72
+ "segment_count_current_delta": 98,
73
+ "segment_count_peak": 1116,
74
+ "segment_count_peak_delta": 98,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T190754288383Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:54.320885Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190754288383Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:85e3e8ed2bfb9ab468796aae78baa71b03b8237dab69340a405d06da59f3e7ca
3
+ size 1673137
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190754565485Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72076615168,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 95064244224,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 72076615168,
8
+ "active_bytes_peak": 95064244224,
9
+ "allocated_bytes_current": 72076615168,
10
+ "allocated_bytes_peak": 95064244224,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 2131108352,
13
+ "inactive_split_bytes_peak": 5056163328,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 100709433344,
21
+ "reserved_bytes_peak": 100709433344,
22
+ "segment_count_current": 1116,
23
+ "segment_count_peak": 1116,
24
+ "timestamp_utc": "2026-08-26T19:07:54.565074Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72076615168,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 95064244224,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 72076615168,
33
+ "active_bytes_peak": 95064244224,
34
+ "allocated_bytes_current": 72076615168,
35
+ "allocated_bytes_peak": 95064244224,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 2131108352,
38
+ "inactive_split_bytes_peak": 5056163328,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 100709433344,
46
+ "reserved_bytes_peak": 100709433344,
47
+ "segment_count_current": 1116,
48
+ "segment_count_peak": 1116,
49
+ "timestamp_utc": "2026-08-26T19:07:54.324586Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 2131108352,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 5056163328,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 100709433344,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 100709433344,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1116,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1116,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T190754565485Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:54.594182Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190754565485Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5fce0c4117776b38d996deabb0758da5b8b39d2c492b81ae68bc4504910185c4
3
+ size 1669731
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190756149994Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72093658624,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 99473075200,
5
+ "active_bytes_peak_delta": 4408830976,
6
+ "after": {
7
+ "active_bytes_current": 72093658624,
8
+ "active_bytes_peak": 99473075200,
9
+ "allocated_bytes_current": 72093658624,
10
+ "allocated_bytes_peak": 95069056000,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 2030178816,
13
+ "inactive_split_bytes_peak": 5860045312,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 108303220736,
21
+ "reserved_bytes_peak": 108303220736,
22
+ "segment_count_current": 1158,
23
+ "segment_count_peak": 1158,
24
+ "timestamp_utc": "2026-08-26T19:07:56.149563Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72093658624,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 95069056000,
30
+ "allocated_bytes_peak_delta": 4811776,
31
+ "before": {
32
+ "active_bytes_current": 72076615168,
33
+ "active_bytes_peak": 95064244224,
34
+ "allocated_bytes_current": 72076615168,
35
+ "allocated_bytes_peak": 95064244224,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 2131108352,
38
+ "inactive_split_bytes_peak": 5056163328,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 100709433344,
46
+ "reserved_bytes_peak": 100709433344,
47
+ "segment_count_current": 1116,
48
+ "segment_count_peak": 1116,
49
+ "timestamp_utc": "2026-08-26T19:07:54.597543Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 2030178816,
55
+ "inactive_split_bytes_current_delta": -100929536,
56
+ "inactive_split_bytes_peak": 5860045312,
57
+ "inactive_split_bytes_peak_delta": 803881984,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 108303220736,
68
+ "reserved_bytes_current_delta": 7593787392,
69
+ "reserved_bytes_peak": 108303220736,
70
+ "reserved_bytes_peak_delta": 7593787392,
71
+ "segment_count_current": 1158,
72
+ "segment_count_current_delta": 42,
73
+ "segment_count_peak": 1158,
74
+ "segment_count_peak_delta": 42,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T190756149994Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:56.185386Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190756149994Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:036e2442bcd51d9d76f99f3882c13d6cab46d83cc9c568862a3e08e8c9aa24f8
3
+ size 1929444
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190756378065Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72093658624,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 99473075200,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 72093658624,
8
+ "active_bytes_peak": 99473075200,
9
+ "allocated_bytes_current": 72093658624,
10
+ "allocated_bytes_peak": 95069056000,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 2030178816,
13
+ "inactive_split_bytes_peak": 5860045312,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 108303220736,
21
+ "reserved_bytes_peak": 108303220736,
22
+ "segment_count_current": 1158,
23
+ "segment_count_peak": 1158,
24
+ "timestamp_utc": "2026-08-26T19:07:56.377662Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72093658624,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 95069056000,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 72093658624,
33
+ "active_bytes_peak": 99473075200,
34
+ "allocated_bytes_current": 72093658624,
35
+ "allocated_bytes_peak": 95069056000,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 2030178816,
38
+ "inactive_split_bytes_peak": 5860045312,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 108303220736,
46
+ "reserved_bytes_peak": 108303220736,
47
+ "segment_count_current": 1158,
48
+ "segment_count_peak": 1158,
49
+ "timestamp_utc": "2026-08-26T19:07:56.189138Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 2030178816,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 5860045312,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 108303220736,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 108303220736,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1158,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1158,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T190756378065Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:56.384905Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190756378065Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2992fad4156e1d350878212bb7a63982fbb2b56096c61356234d0445f67d7bad
3
+ size 304872
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190756568213Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72093658624,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 99473075200,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 72093658624,
8
+ "active_bytes_peak": 99473075200,
9
+ "allocated_bytes_current": 72093658624,
10
+ "allocated_bytes_peak": 95069056000,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 2030178816,
13
+ "inactive_split_bytes_peak": 5860045312,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 108303220736,
21
+ "reserved_bytes_peak": 108303220736,
22
+ "segment_count_current": 1158,
23
+ "segment_count_peak": 1158,
24
+ "timestamp_utc": "2026-08-26T19:07:56.567820Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72093658624,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 95069056000,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 72093658624,
33
+ "active_bytes_peak": 99473075200,
34
+ "allocated_bytes_current": 72093658624,
35
+ "allocated_bytes_peak": 95069056000,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 2030178816,
38
+ "inactive_split_bytes_peak": 5860045312,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 108303220736,
46
+ "reserved_bytes_peak": 108303220736,
47
+ "segment_count_current": 1158,
48
+ "segment_count_peak": 1158,
49
+ "timestamp_utc": "2026-08-26T19:07:56.385924Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 2030178816,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 5860045312,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 108303220736,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 108303220736,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1158,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1158,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T190756568213Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:56.575099Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190756568213Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:39fae1522f7b4a64dd46e1d572997c4344de54a5d9f75590fcb817f089c395ee
3
+ size 304841
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190753657101Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72076615168,
3
+ "active_bytes_current_delta": 31390646784,
4
+ "active_bytes_peak": 74041179136,
5
+ "active_bytes_peak_delta": 33317462016,
6
+ "after": {
7
+ "active_bytes_current": 72076615168,
8
+ "active_bytes_peak": 74041179136,
9
+ "allocated_bytes_current": 72076615168,
10
+ "allocated_bytes_peak": 74041179136,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 1835409920,
13
+ "inactive_split_bytes_peak": 5056163328,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 84255178752,
21
+ "reserved_bytes_peak": 84255178752,
22
+ "segment_count_current": 1018,
23
+ "segment_count_peak": 1018,
24
+ "timestamp_utc": "2026-08-26T19:07:53.656602Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72076615168,
28
+ "allocated_bytes_current_delta": 31390646784,
29
+ "allocated_bytes_peak": 74041179136,
30
+ "allocated_bytes_peak_delta": 33317462016,
31
+ "before": {
32
+ "active_bytes_current": 40685968384,
33
+ "active_bytes_peak": 40723717120,
34
+ "allocated_bytes_current": 40685968384,
35
+ "allocated_bytes_peak": 40723717120,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 508388352,
38
+ "inactive_split_bytes_peak": 728589312,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 43864031232,
46
+ "reserved_bytes_peak": 43864031232,
47
+ "segment_count_current": 261,
48
+ "segment_count_peak": 261,
49
+ "timestamp_utc": "2026-08-26T19:07:14.578992Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1835409920,
55
+ "inactive_split_bytes_current_delta": 1327021568,
56
+ "inactive_split_bytes_peak": 5056163328,
57
+ "inactive_split_bytes_peak_delta": 4327574016,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 84255178752,
68
+ "reserved_bytes_current_delta": 40391147520,
69
+ "reserved_bytes_peak": 84255178752,
70
+ "reserved_bytes_peak_delta": 40391147520,
71
+ "segment_count_current": 1018,
72
+ "segment_count_current_delta": 757,
73
+ "segment_count_peak": 1018,
74
+ "segment_count_peak_delta": 757,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T190753657101Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:53.693328Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190753657101Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c67267c2569542f43a21e5a0d76ce414c2aea750e7ab0709b11e2b441f3193fd
3
+ size 2049975
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190754288388Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72076615168,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 95064244224,
5
+ "active_bytes_peak_delta": 21023065088,
6
+ "after": {
7
+ "active_bytes_current": 72076615168,
8
+ "active_bytes_peak": 95064244224,
9
+ "allocated_bytes_current": 72076615168,
10
+ "allocated_bytes_peak": 95064244224,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 2131108352,
13
+ "inactive_split_bytes_peak": 5056163328,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 99828629504,
21
+ "reserved_bytes_peak": 99828629504,
22
+ "segment_count_current": 1113,
23
+ "segment_count_peak": 1113,
24
+ "timestamp_utc": "2026-08-26T19:07:54.287945Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72076615168,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 95064244224,
30
+ "allocated_bytes_peak_delta": 21023065088,
31
+ "before": {
32
+ "active_bytes_current": 72076615168,
33
+ "active_bytes_peak": 74041179136,
34
+ "allocated_bytes_current": 72076615168,
35
+ "allocated_bytes_peak": 74041179136,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 1835409920,
38
+ "inactive_split_bytes_peak": 5056163328,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 84255178752,
46
+ "reserved_bytes_peak": 84255178752,
47
+ "segment_count_current": 1018,
48
+ "segment_count_peak": 1018,
49
+ "timestamp_utc": "2026-08-26T19:07:53.697869Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 2131108352,
55
+ "inactive_split_bytes_current_delta": 295698432,
56
+ "inactive_split_bytes_peak": 5056163328,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 99828629504,
68
+ "reserved_bytes_current_delta": 15573450752,
69
+ "reserved_bytes_peak": 99828629504,
70
+ "reserved_bytes_peak_delta": 15573450752,
71
+ "segment_count_current": 1113,
72
+ "segment_count_current_delta": 95,
73
+ "segment_count_peak": 1113,
74
+ "segment_count_peak_delta": 95,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T190754288388Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:54.320175Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190754288388Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:910bc477e0ff1b4ba1c368ef808734a91bbdb16149c456dc613ad8a0de656026
3
+ size 1669041
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190754566103Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72076615168,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 95064244224,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 72076615168,
8
+ "active_bytes_peak": 95064244224,
9
+ "allocated_bytes_current": 72076615168,
10
+ "allocated_bytes_peak": 95064244224,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 2131108352,
13
+ "inactive_split_bytes_peak": 5056163328,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 99828629504,
21
+ "reserved_bytes_peak": 99828629504,
22
+ "segment_count_current": 1113,
23
+ "segment_count_peak": 1113,
24
+ "timestamp_utc": "2026-08-26T19:07:54.565581Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72076615168,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 95064244224,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 72076615168,
33
+ "active_bytes_peak": 95064244224,
34
+ "allocated_bytes_current": 72076615168,
35
+ "allocated_bytes_peak": 95064244224,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 2131108352,
38
+ "inactive_split_bytes_peak": 5056163328,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 99828629504,
46
+ "reserved_bytes_peak": 99828629504,
47
+ "segment_count_current": 1113,
48
+ "segment_count_peak": 1113,
49
+ "timestamp_utc": "2026-08-26T19:07:54.324156Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 2131108352,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 5056163328,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 99828629504,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 99828629504,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1113,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1113,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T190754566103Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:54.600995Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190754566103Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3be16039d71d6c2058771a91dcac57af5c4f4c1798269998227160f87b9b348c
3
+ size 1659589
echo/tierbmem/tierbmem_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap1p125__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190756150024Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 72093658624,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 99473075200,
5
+ "active_bytes_peak_delta": 4408830976,
6
+ "after": {
7
+ "active_bytes_current": 72093658624,
8
+ "active_bytes_peak": 99473075200,
9
+ "allocated_bytes_current": 72093658624,
10
+ "allocated_bytes_peak": 95069056000,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 2030178816,
13
+ "inactive_split_bytes_peak": 5803422208,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 108303220736,
21
+ "reserved_bytes_peak": 108303220736,
22
+ "segment_count_current": 1158,
23
+ "segment_count_peak": 1158,
24
+ "timestamp_utc": "2026-08-26T19:07:56.149558Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 72093658624,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 95069056000,
30
+ "allocated_bytes_peak_delta": 4811776,
31
+ "before": {
32
+ "active_bytes_current": 72076615168,
33
+ "active_bytes_peak": 95064244224,
34
+ "allocated_bytes_current": 72076615168,
35
+ "allocated_bytes_peak": 95064244224,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 2131108352,
38
+ "inactive_split_bytes_peak": 5056163328,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 99828629504,
46
+ "reserved_bytes_peak": 99828629504,
47
+ "segment_count_current": 1113,
48
+ "segment_count_peak": 1113,
49
+ "timestamp_utc": "2026-08-26T19:07:54.605114Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 2030178816,
55
+ "inactive_split_bytes_current_delta": -100929536,
56
+ "inactive_split_bytes_peak": 5803422208,
57
+ "inactive_split_bytes_peak_delta": 747258880,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 108303220736,
68
+ "reserved_bytes_current_delta": 8474591232,
69
+ "reserved_bytes_peak": 108303220736,
70
+ "reserved_bytes_peak_delta": 8474591232,
71
+ "segment_count_current": 1158,
72
+ "segment_count_current_delta": 45,
73
+ "segment_count_peak": 1158,
74
+ "segment_count_peak_delta": 45,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T190756150024Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:07:56.186479Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }