AbhishekGhosh1998 commited on
Commit
468c662
·
verified ·
1 Parent(s): c35cd98

granularity results echo/tierbmem @ 2026-08-26T19:10:38Z from sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 (part 2)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190402080527Z.json +79 -0
  2. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190402080527Z.pickle +3 -0
  3. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190402816632Z.json +79 -0
  4. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190402816632Z.pickle +3 -0
  5. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190403545207Z.json +79 -0
  6. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190403545207Z.pickle +3 -0
  7. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190405730680Z.json +79 -0
  8. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190405730680Z.pickle +3 -0
  9. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190406419097Z.json +79 -0
  10. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190406419097Z.pickle +3 -0
  11. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190407024521Z.json +79 -0
  12. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190407024521Z.pickle +3 -0
  13. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190402080520Z.json +79 -0
  14. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190402080520Z.pickle +3 -0
  15. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190402816576Z.json +79 -0
  16. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190402816576Z.pickle +3 -0
  17. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190403545145Z.json +79 -0
  18. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190403545145Z.pickle +3 -0
  19. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190405730472Z.json +79 -0
  20. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190405730472Z.pickle +3 -0
  21. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190406419060Z.json +79 -0
  22. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190406419060Z.pickle +3 -0
  23. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190407024444Z.json +79 -0
  24. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190407024444Z.pickle +3 -0
  25. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190402080508Z.json +79 -0
  26. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190402080508Z.pickle +3 -0
  27. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190402816632Z.json +79 -0
  28. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190402816632Z.pickle +3 -0
  29. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190403545151Z.json +79 -0
  30. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190403545151Z.pickle +3 -0
  31. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190405730376Z.json +79 -0
  32. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190405730376Z.pickle +3 -0
  33. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190406419077Z.json +79 -0
  34. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190406419077Z.pickle +3 -0
  35. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190407024459Z.json +79 -0
  36. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190407024459Z.pickle +3 -0
  37. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190402080520Z.json +79 -0
  38. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190402080520Z.pickle +3 -0
  39. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190402816603Z.json +79 -0
  40. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190402816603Z.pickle +3 -0
  41. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190403545196Z.json +79 -0
  42. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190403545196Z.pickle +3 -0
  43. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190405730411Z.json +79 -0
  44. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190405730411Z.pickle +3 -0
  45. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190406419136Z.json +79 -0
  46. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190406419136Z.pickle +3 -0
  47. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190407024499Z.json +79 -0
  48. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190407024499Z.pickle +3 -0
  49. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env +61 -0
  50. echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log +0 -0
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190402080527Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42478680576,
3
+ "active_bytes_current_delta": 19430297088,
4
+ "active_bytes_peak": 113227885568,
5
+ "active_bytes_peak_delta": 90166919168,
6
+ "after": {
7
+ "active_bytes_current": 42478680576,
8
+ "active_bytes_peak": 113227885568,
9
+ "allocated_bytes_current": 42478680576,
10
+ "allocated_bytes_peak": 113227885568,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 18456167936,
13
+ "inactive_split_bytes_peak": 30415799296,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 117411151872,
21
+ "reserved_bytes_peak": 117411151872,
22
+ "segment_count_current": 920,
23
+ "segment_count_peak": 920,
24
+ "timestamp_utc": "2026-08-26T19:04:02.080066Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42478680576,
28
+ "allocated_bytes_current_delta": 19430297088,
29
+ "allocated_bytes_peak": 113227885568,
30
+ "allocated_bytes_peak_delta": 90166919168,
31
+ "before": {
32
+ "active_bytes_current": 23048383488,
33
+ "active_bytes_peak": 23060966400,
34
+ "allocated_bytes_current": 23048383488,
35
+ "allocated_bytes_peak": 23060966400,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 376804352,
38
+ "inactive_split_bytes_peak": 883235328,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 25623003136,
46
+ "reserved_bytes_peak": 25623003136,
47
+ "segment_count_current": 551,
48
+ "segment_count_peak": 551,
49
+ "timestamp_utc": "2026-08-26T19:03:23.745553Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18456167936,
55
+ "inactive_split_bytes_current_delta": 18079363584,
56
+ "inactive_split_bytes_peak": 30415799296,
57
+ "inactive_split_bytes_peak_delta": 29532563968,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 117411151872,
68
+ "reserved_bytes_current_delta": 91788148736,
69
+ "reserved_bytes_peak": 117411151872,
70
+ "reserved_bytes_peak_delta": 91788148736,
71
+ "segment_count_current": 920,
72
+ "segment_count_current_delta": 369,
73
+ "segment_count_peak": 920,
74
+ "segment_count_peak_delta": 369,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260826T190402080527Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:02.162509Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000000_20260826T190402080527Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d83729cb37450461395f2b17a82bced8dc5205c1a1dfc7f01154f3a556b77380
3
+ size 4114465
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190402816632Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42478680576,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 127729292800,
5
+ "active_bytes_peak_delta": 14501407232,
6
+ "after": {
7
+ "active_bytes_current": 42478680576,
8
+ "active_bytes_peak": 127729292800,
9
+ "allocated_bytes_current": 42478680576,
10
+ "allocated_bytes_peak": 127452468736,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 18548442624,
13
+ "inactive_split_bytes_peak": 30684399104,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 129807417344,
21
+ "reserved_bytes_peak": 129807417344,
22
+ "segment_count_current": 931,
23
+ "segment_count_peak": 931,
24
+ "timestamp_utc": "2026-08-26T19:04:02.816236Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42478680576,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 127452468736,
30
+ "allocated_bytes_peak_delta": 14224583168,
31
+ "before": {
32
+ "active_bytes_current": 42478680576,
33
+ "active_bytes_peak": 113227885568,
34
+ "allocated_bytes_current": 42478680576,
35
+ "allocated_bytes_peak": 113227885568,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 18456167936,
38
+ "inactive_split_bytes_peak": 30415799296,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 117411151872,
46
+ "reserved_bytes_peak": 117411151872,
47
+ "segment_count_current": 920,
48
+ "segment_count_peak": 920,
49
+ "timestamp_utc": "2026-08-26T19:04:02.173774Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18548442624,
55
+ "inactive_split_bytes_current_delta": 92274688,
56
+ "inactive_split_bytes_peak": 30684399104,
57
+ "inactive_split_bytes_peak_delta": 268599808,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 129807417344,
68
+ "reserved_bytes_current_delta": 12396265472,
69
+ "reserved_bytes_peak": 129807417344,
70
+ "reserved_bytes_peak_delta": 12396265472,
71
+ "segment_count_current": 931,
72
+ "segment_count_current_delta": 11,
73
+ "segment_count_peak": 931,
74
+ "segment_count_peak_delta": 11,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260826T190402816632Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:02.891028Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000001_20260826T190402816632Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ebe4fbadd82e9e91e33d1c17ef2c8d74cfd0adc06e63be0dc3e564db86ac3b95
3
+ size 4503524
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190403545207Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42478680576,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 127731652096,
5
+ "active_bytes_peak_delta": 2359296,
6
+ "after": {
7
+ "active_bytes_current": 42478680576,
8
+ "active_bytes_peak": 127731652096,
9
+ "allocated_bytes_current": 42478680576,
10
+ "allocated_bytes_peak": 127454828032,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 18458265088,
13
+ "inactive_split_bytes_peak": 30684399104,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 129807417344,
21
+ "reserved_bytes_peak": 129807417344,
22
+ "segment_count_current": 931,
23
+ "segment_count_peak": 931,
24
+ "timestamp_utc": "2026-08-26T19:04:03.544793Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42478680576,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 127454828032,
30
+ "allocated_bytes_peak_delta": 2359296,
31
+ "before": {
32
+ "active_bytes_current": 42478680576,
33
+ "active_bytes_peak": 127729292800,
34
+ "allocated_bytes_current": 42478680576,
35
+ "allocated_bytes_peak": 127452468736,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 18548442624,
38
+ "inactive_split_bytes_peak": 30684399104,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 129807417344,
46
+ "reserved_bytes_peak": 129807417344,
47
+ "segment_count_current": 931,
48
+ "segment_count_peak": 931,
49
+ "timestamp_utc": "2026-08-26T19:04:02.899770Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18458265088,
55
+ "inactive_split_bytes_current_delta": -90177536,
56
+ "inactive_split_bytes_peak": 30684399104,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 129807417344,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 129807417344,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 931,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 931,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260826T190403545207Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:03.620923Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260826T190403545207Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fac8cfeefd3e11c9761bb1030a4b904986da1d50ce422eb6127c83e2cd002336
3
+ size 4520747
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190405730680Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42496641536,
3
+ "active_bytes_current_delta": 17960960,
4
+ "active_bytes_peak": 130557001728,
5
+ "active_bytes_peak_delta": 2825349632,
6
+ "after": {
7
+ "active_bytes_current": 42496641536,
8
+ "active_bytes_peak": 130557001728,
9
+ "allocated_bytes_current": 42496641536,
10
+ "allocated_bytes_peak": 127454828032,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 20610856448,
13
+ "inactive_split_bytes_peak": 46659659264,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 155212316672,
21
+ "reserved_bytes_peak": 155212316672,
22
+ "segment_count_current": 1130,
23
+ "segment_count_peak": 1130,
24
+ "timestamp_utc": "2026-08-26T19:04:05.730047Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42496641536,
28
+ "allocated_bytes_current_delta": 17960960,
29
+ "allocated_bytes_peak": 127454828032,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42478680576,
33
+ "active_bytes_peak": 127731652096,
34
+ "allocated_bytes_current": 42478680576,
35
+ "allocated_bytes_peak": 127454828032,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 18458265088,
38
+ "inactive_split_bytes_peak": 30684399104,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 129807417344,
46
+ "reserved_bytes_peak": 129807417344,
47
+ "segment_count_current": 931,
48
+ "segment_count_peak": 931,
49
+ "timestamp_utc": "2026-08-26T19:04:03.629839Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 20610856448,
55
+ "inactive_split_bytes_current_delta": 2152591360,
56
+ "inactive_split_bytes_peak": 46659659264,
57
+ "inactive_split_bytes_peak_delta": 15975260160,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 155212316672,
68
+ "reserved_bytes_current_delta": 25404899328,
69
+ "reserved_bytes_peak": 155212316672,
70
+ "reserved_bytes_peak_delta": 25404899328,
71
+ "segment_count_current": 1130,
72
+ "segment_count_current_delta": 199,
73
+ "segment_count_peak": 1130,
74
+ "segment_count_peak_delta": 199,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260826T190405730680Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:05.818896Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260826T190405730680Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:10ac130196532840e49750a16588a05035df2819076786300a6423ec1be2210e
3
+ size 4737184
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190406419097Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42496641536,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 130557001728,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 42496641536,
8
+ "active_bytes_peak": 130557001728,
9
+ "allocated_bytes_current": 42496641536,
10
+ "allocated_bytes_peak": 127454828032,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 20610856448,
13
+ "inactive_split_bytes_peak": 46659659264,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 155212316672,
21
+ "reserved_bytes_peak": 155212316672,
22
+ "segment_count_current": 1130,
23
+ "segment_count_peak": 1130,
24
+ "timestamp_utc": "2026-08-26T19:04:06.418698Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42496641536,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 127454828032,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42496641536,
33
+ "active_bytes_peak": 130557001728,
34
+ "allocated_bytes_current": 42496641536,
35
+ "allocated_bytes_peak": 127454828032,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 20610856448,
38
+ "inactive_split_bytes_peak": 46659659264,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 155212316672,
46
+ "reserved_bytes_peak": 155212316672,
47
+ "segment_count_current": 1130,
48
+ "segment_count_peak": 1130,
49
+ "timestamp_utc": "2026-08-26T19:04:05.828949Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 20610856448,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 46659659264,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 155212316672,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 155212316672,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1130,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1130,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260826T190406419097Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:06.429348Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260826T190406419097Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:18b40bf0480d107b9ef1624b9ce4794741d1aa07c358c1fc50687acd34ec779e
3
+ size 381106
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190407024521Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42496641536,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 130557001728,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 42496641536,
8
+ "active_bytes_peak": 130557001728,
9
+ "allocated_bytes_current": 42496641536,
10
+ "allocated_bytes_peak": 127454828032,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 20610856448,
13
+ "inactive_split_bytes_peak": 46659659264,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 155212316672,
21
+ "reserved_bytes_peak": 155212316672,
22
+ "segment_count_current": 1130,
23
+ "segment_count_peak": 1130,
24
+ "timestamp_utc": "2026-08-26T19:04:07.024122Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42496641536,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 127454828032,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42496641536,
33
+ "active_bytes_peak": 130557001728,
34
+ "allocated_bytes_current": 42496641536,
35
+ "allocated_bytes_peak": 127454828032,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 20610856448,
38
+ "inactive_split_bytes_peak": 46659659264,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 155212316672,
46
+ "reserved_bytes_peak": 155212316672,
47
+ "segment_count_current": 1130,
48
+ "segment_count_peak": 1130,
49
+ "timestamp_utc": "2026-08-26T19:04:06.430441Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 20610856448,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 46659659264,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 155212316672,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 155212316672,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1130,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1130,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260826T190407024521Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:07.034324Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000005_20260826T190407024521Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c81cf1f8019b801f8aa391c3519ce153b5a72c6ec7200125305317d1c965999
3
+ size 381106
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190402080520Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42487518720,
3
+ "active_bytes_current_delta": 19434335744,
4
+ "active_bytes_peak": 113240445952,
5
+ "active_bytes_peak_delta": 90174680064,
6
+ "after": {
7
+ "active_bytes_current": 42487518720,
8
+ "active_bytes_peak": 113240445952,
9
+ "allocated_bytes_current": 42487518720,
10
+ "allocated_bytes_peak": 113240445952,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 18376026624,
13
+ "inactive_split_bytes_peak": 30425401344,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 117409054720,
21
+ "reserved_bytes_peak": 117409054720,
22
+ "segment_count_current": 919,
23
+ "segment_count_peak": 919,
24
+ "timestamp_utc": "2026-08-26T19:04:02.080015Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42487518720,
28
+ "allocated_bytes_current_delta": 19434335744,
29
+ "allocated_bytes_peak": 113240445952,
30
+ "allocated_bytes_peak_delta": 90174680064,
31
+ "before": {
32
+ "active_bytes_current": 23053182976,
33
+ "active_bytes_peak": 23065765888,
34
+ "allocated_bytes_current": 23053182976,
35
+ "allocated_bytes_peak": 23065765888,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 462182400,
38
+ "inactive_split_bytes_peak": 883235328,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 25623003136,
46
+ "reserved_bytes_peak": 25623003136,
47
+ "segment_count_current": 551,
48
+ "segment_count_peak": 551,
49
+ "timestamp_utc": "2026-08-26T19:03:23.745648Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18376026624,
55
+ "inactive_split_bytes_current_delta": 17913844224,
56
+ "inactive_split_bytes_peak": 30425401344,
57
+ "inactive_split_bytes_peak_delta": 29542166016,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 117409054720,
68
+ "reserved_bytes_current_delta": 91786051584,
69
+ "reserved_bytes_peak": 117409054720,
70
+ "reserved_bytes_peak_delta": 91786051584,
71
+ "segment_count_current": 919,
72
+ "segment_count_current_delta": 368,
73
+ "segment_count_peak": 919,
74
+ "segment_count_peak_delta": 368,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260826T190402080520Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:02.166044Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260826T190402080520Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:56b903ccb299ef35670c726458cd25437f737f6bf7790813de00992262510475
3
+ size 4134331
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190402816576Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42487518720,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 127585480192,
5
+ "active_bytes_peak_delta": 14345034240,
6
+ "after": {
7
+ "active_bytes_current": 42487518720,
8
+ "active_bytes_peak": 127585480192,
9
+ "allocated_bytes_current": 42487518720,
10
+ "allocated_bytes_peak": 127447068160,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 18468301312,
13
+ "inactive_split_bytes_peak": 30425401344,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 131946512384,
21
+ "reserved_bytes_peak": 131946512384,
22
+ "segment_count_current": 928,
23
+ "segment_count_peak": 928,
24
+ "timestamp_utc": "2026-08-26T19:04:02.816170Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42487518720,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 127447068160,
30
+ "allocated_bytes_peak_delta": 14206622208,
31
+ "before": {
32
+ "active_bytes_current": 42487518720,
33
+ "active_bytes_peak": 113240445952,
34
+ "allocated_bytes_current": 42487518720,
35
+ "allocated_bytes_peak": 113240445952,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 18376026624,
38
+ "inactive_split_bytes_peak": 30425401344,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 117409054720,
46
+ "reserved_bytes_peak": 117409054720,
47
+ "segment_count_current": 919,
48
+ "segment_count_peak": 919,
49
+ "timestamp_utc": "2026-08-26T19:04:02.178637Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18468301312,
55
+ "inactive_split_bytes_current_delta": 92274688,
56
+ "inactive_split_bytes_peak": 30425401344,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 131946512384,
68
+ "reserved_bytes_current_delta": 14537457664,
69
+ "reserved_bytes_peak": 131946512384,
70
+ "reserved_bytes_peak_delta": 14537457664,
71
+ "segment_count_current": 928,
72
+ "segment_count_current_delta": 9,
73
+ "segment_count_peak": 928,
74
+ "segment_count_peak_delta": 9,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260826T190402816576Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:02.892646Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260826T190402816576Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:96de714f4a3d9f18d4e49013712ee500a525c925beb064029d5c21ddadb9cbe1
3
+ size 4529243
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190403545145Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42487518720,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 127718985216,
5
+ "active_bytes_peak_delta": 133505024,
6
+ "after": {
7
+ "active_bytes_current": 42487518720,
8
+ "active_bytes_peak": 127718985216,
9
+ "allocated_bytes_current": 42487518720,
10
+ "allocated_bytes_peak": 127447068160,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 18470398464,
13
+ "inactive_split_bytes_peak": 31822153216,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 131946512384,
21
+ "reserved_bytes_peak": 131946512384,
22
+ "segment_count_current": 928,
23
+ "segment_count_peak": 928,
24
+ "timestamp_utc": "2026-08-26T19:04:03.544740Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42487518720,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 127447068160,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42487518720,
33
+ "active_bytes_peak": 127585480192,
34
+ "allocated_bytes_current": 42487518720,
35
+ "allocated_bytes_peak": 127447068160,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 18468301312,
38
+ "inactive_split_bytes_peak": 30425401344,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 131946512384,
46
+ "reserved_bytes_peak": 131946512384,
47
+ "segment_count_current": 928,
48
+ "segment_count_peak": 928,
49
+ "timestamp_utc": "2026-08-26T19:04:02.902746Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18470398464,
55
+ "inactive_split_bytes_current_delta": 2097152,
56
+ "inactive_split_bytes_peak": 31822153216,
57
+ "inactive_split_bytes_peak_delta": 1396751872,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 131946512384,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 131946512384,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 928,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 928,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260826T190403545145Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:03.619957Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260826T190403545145Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c8afa962f2fe520b956cebd932c0a11446c25f3b7e7950d89aa43ec0c43724a6
3
+ size 4545972
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190405730472Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42505479680,
3
+ "active_bytes_current_delta": 17960960,
4
+ "active_bytes_peak": 130565839872,
5
+ "active_bytes_peak_delta": 2846854656,
6
+ "after": {
7
+ "active_bytes_current": 42505479680,
8
+ "active_bytes_peak": 130565839872,
9
+ "allocated_bytes_current": 42505479680,
10
+ "allocated_bytes_peak": 127447068160,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 20530715136,
13
+ "inactive_split_bytes_peak": 46669695488,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 155233288192,
21
+ "reserved_bytes_peak": 155233288192,
22
+ "segment_count_current": 1097,
23
+ "segment_count_peak": 1097,
24
+ "timestamp_utc": "2026-08-26T19:04:05.729868Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42505479680,
28
+ "allocated_bytes_current_delta": 17960960,
29
+ "allocated_bytes_peak": 127447068160,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42487518720,
33
+ "active_bytes_peak": 127718985216,
34
+ "allocated_bytes_current": 42487518720,
35
+ "allocated_bytes_peak": 127447068160,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 18470398464,
38
+ "inactive_split_bytes_peak": 31822153216,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 131946512384,
46
+ "reserved_bytes_peak": 131946512384,
47
+ "segment_count_current": 928,
48
+ "segment_count_peak": 928,
49
+ "timestamp_utc": "2026-08-26T19:04:03.629418Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 20530715136,
55
+ "inactive_split_bytes_current_delta": 2060316672,
56
+ "inactive_split_bytes_peak": 46669695488,
57
+ "inactive_split_bytes_peak_delta": 14847542272,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 155233288192,
68
+ "reserved_bytes_current_delta": 23286775808,
69
+ "reserved_bytes_peak": 155233288192,
70
+ "reserved_bytes_peak_delta": 23286775808,
71
+ "segment_count_current": 1097,
72
+ "segment_count_current_delta": 169,
73
+ "segment_count_peak": 1097,
74
+ "segment_count_peak_delta": 169,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260826T190405730472Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:05.816882Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260826T190405730472Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:624ccf54acb5bc1133e019f5644605beeb1cdb228b6a43a6a0670036ce80281d
3
+ size 4761905
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190406419060Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42505479680,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 130565839872,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 42505479680,
8
+ "active_bytes_peak": 130565839872,
9
+ "allocated_bytes_current": 42505479680,
10
+ "allocated_bytes_peak": 127447068160,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 20530715136,
13
+ "inactive_split_bytes_peak": 46669695488,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 155233288192,
21
+ "reserved_bytes_peak": 155233288192,
22
+ "segment_count_current": 1097,
23
+ "segment_count_peak": 1097,
24
+ "timestamp_utc": "2026-08-26T19:04:06.418658Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42505479680,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 127447068160,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42505479680,
33
+ "active_bytes_peak": 130565839872,
34
+ "allocated_bytes_current": 42505479680,
35
+ "allocated_bytes_peak": 127447068160,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 20530715136,
38
+ "inactive_split_bytes_peak": 46669695488,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 155233288192,
46
+ "reserved_bytes_peak": 155233288192,
47
+ "segment_count_current": 1097,
48
+ "segment_count_peak": 1097,
49
+ "timestamp_utc": "2026-08-26T19:04:05.829885Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 20530715136,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 46669695488,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 155233288192,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 155233288192,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1097,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1097,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260826T190406419060Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:06.429246Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260826T190406419060Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:01890f6d738e8cc73fafbc7c5b378623fb52a98b01996eeb5a9055b6de5cf06f
3
+ size 403620
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190407024444Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42505479680,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 130565839872,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 42505479680,
8
+ "active_bytes_peak": 130565839872,
9
+ "allocated_bytes_current": 42505479680,
10
+ "allocated_bytes_peak": 127447068160,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 20530715136,
13
+ "inactive_split_bytes_peak": 46669695488,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 155233288192,
21
+ "reserved_bytes_peak": 155233288192,
22
+ "segment_count_current": 1097,
23
+ "segment_count_peak": 1097,
24
+ "timestamp_utc": "2026-08-26T19:04:07.024063Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42505479680,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 127447068160,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42505479680,
33
+ "active_bytes_peak": 130565839872,
34
+ "allocated_bytes_current": 42505479680,
35
+ "allocated_bytes_peak": 127447068160,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 20530715136,
38
+ "inactive_split_bytes_peak": 46669695488,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 155233288192,
46
+ "reserved_bytes_peak": 155233288192,
47
+ "segment_count_current": 1097,
48
+ "segment_count_peak": 1097,
49
+ "timestamp_utc": "2026-08-26T19:04:06.430320Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 20530715136,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 46669695488,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 155233288192,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 155233288192,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1097,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1097,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260826T190407024444Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:07.033441Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000005_20260826T190407024444Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:28828d5ebd6237774c5b03e0400603dab08621171960bff1a06def72d36e5682
3
+ size 403620
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190402080508Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42485283328,
3
+ "active_bytes_current_delta": 19439536128,
4
+ "active_bytes_peak": 113234178048,
5
+ "active_bytes_peak_delta": 90175847936,
6
+ "after": {
7
+ "active_bytes_current": 42485283328,
8
+ "active_bytes_peak": 113234178048,
9
+ "allocated_bytes_current": 42485283328,
10
+ "allocated_bytes_peak": 113234178048,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 18608948736,
13
+ "inactive_split_bytes_peak": 30468116992,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 117549563904,
21
+ "reserved_bytes_peak": 117549563904,
22
+ "segment_count_current": 926,
23
+ "segment_count_peak": 926,
24
+ "timestamp_utc": "2026-08-26T19:04:02.079986Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42485283328,
28
+ "allocated_bytes_current_delta": 19439536128,
29
+ "allocated_bytes_peak": 113234178048,
30
+ "allocated_bytes_peak_delta": 90175847936,
31
+ "before": {
32
+ "active_bytes_current": 23045747200,
33
+ "active_bytes_peak": 23058330112,
34
+ "allocated_bytes_current": 23045747200,
35
+ "allocated_bytes_peak": 23058330112,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 358469120,
38
+ "inactive_split_bytes_peak": 883235328,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 25761415168,
46
+ "reserved_bytes_peak": 25761415168,
47
+ "segment_count_current": 557,
48
+ "segment_count_peak": 557,
49
+ "timestamp_utc": "2026-08-26T19:03:23.745539Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18608948736,
55
+ "inactive_split_bytes_current_delta": 18250479616,
56
+ "inactive_split_bytes_peak": 30468116992,
57
+ "inactive_split_bytes_peak_delta": 29584881664,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 117549563904,
68
+ "reserved_bytes_current_delta": 91788148736,
69
+ "reserved_bytes_peak": 117549563904,
70
+ "reserved_bytes_peak_delta": 91788148736,
71
+ "segment_count_current": 926,
72
+ "segment_count_current_delta": 369,
73
+ "segment_count_peak": 926,
74
+ "segment_count_peak_delta": 369,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260826T190402080508Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:02.162223Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260826T190402080508Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e5087269156536422fe460c092ada4e54ccaa027b0479bb29957631815242864
3
+ size 4119126
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190402816632Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42485283328,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 127602459136,
5
+ "active_bytes_peak_delta": 14368281088,
6
+ "after": {
7
+ "active_bytes_current": 42485283328,
8
+ "active_bytes_peak": 127602459136,
9
+ "allocated_bytes_current": 42485283328,
10
+ "allocated_bytes_peak": 127464047104,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 18701223424,
13
+ "inactive_split_bytes_peak": 30468116992,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 132095410176,
21
+ "reserved_bytes_peak": 132095410176,
22
+ "segment_count_current": 939,
23
+ "segment_count_peak": 939,
24
+ "timestamp_utc": "2026-08-26T19:04:02.816227Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42485283328,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 127464047104,
30
+ "allocated_bytes_peak_delta": 14229869056,
31
+ "before": {
32
+ "active_bytes_current": 42485283328,
33
+ "active_bytes_peak": 113234178048,
34
+ "allocated_bytes_current": 42485283328,
35
+ "allocated_bytes_peak": 113234178048,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 18608948736,
38
+ "inactive_split_bytes_peak": 30468116992,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 117549563904,
46
+ "reserved_bytes_peak": 117549563904,
47
+ "segment_count_current": 926,
48
+ "segment_count_peak": 926,
49
+ "timestamp_utc": "2026-08-26T19:04:02.174428Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18701223424,
55
+ "inactive_split_bytes_current_delta": 92274688,
56
+ "inactive_split_bytes_peak": 30468116992,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 132095410176,
68
+ "reserved_bytes_current_delta": 14545846272,
69
+ "reserved_bytes_peak": 132095410176,
70
+ "reserved_bytes_peak_delta": 14545846272,
71
+ "segment_count_current": 939,
72
+ "segment_count_current_delta": 13,
73
+ "segment_count_peak": 939,
74
+ "segment_count_peak_delta": 13,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260826T190402816632Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:02.893379Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260826T190402816632Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1a6f365f5a7ce3398fb05d1c6cf8ae6aa3ecdc3210fe7c03d685d4a34d72590b
3
+ size 4509200
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190403545151Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42485283328,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 127741936128,
5
+ "active_bytes_peak_delta": 139476992,
6
+ "after": {
7
+ "active_bytes_current": 42485283328,
8
+ "active_bytes_peak": 127741936128,
9
+ "allocated_bytes_current": 42485283328,
10
+ "allocated_bytes_peak": 127465112064,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 18703320576,
13
+ "inactive_split_bytes_peak": 32325633536,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 132187684864,
21
+ "reserved_bytes_peak": 132187684864,
22
+ "segment_count_current": 940,
23
+ "segment_count_peak": 940,
24
+ "timestamp_utc": "2026-08-26T19:04:03.544749Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42485283328,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 127465112064,
30
+ "allocated_bytes_peak_delta": 1064960,
31
+ "before": {
32
+ "active_bytes_current": 42485283328,
33
+ "active_bytes_peak": 127602459136,
34
+ "allocated_bytes_current": 42485283328,
35
+ "allocated_bytes_peak": 127464047104,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 18701223424,
38
+ "inactive_split_bytes_peak": 30468116992,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 132095410176,
46
+ "reserved_bytes_peak": 132095410176,
47
+ "segment_count_current": 939,
48
+ "segment_count_peak": 939,
49
+ "timestamp_utc": "2026-08-26T19:04:02.903317Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18703320576,
55
+ "inactive_split_bytes_current_delta": 2097152,
56
+ "inactive_split_bytes_peak": 32325633536,
57
+ "inactive_split_bytes_peak_delta": 1857516544,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 132187684864,
68
+ "reserved_bytes_current_delta": 92274688,
69
+ "reserved_bytes_peak": 132187684864,
70
+ "reserved_bytes_peak_delta": 92274688,
71
+ "segment_count_current": 940,
72
+ "segment_count_current_delta": 1,
73
+ "segment_count_peak": 940,
74
+ "segment_count_peak_delta": 1,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260826T190403545151Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:03.620155Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260826T190403545151Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aa49457f319d10134dd62b75318b0f7766bf65ef9e67578bb76f120c9311583e
3
+ size 4526558
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190405730376Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42503244288,
3
+ "active_bytes_current_delta": 17960960,
4
+ "active_bytes_peak": 130563604480,
5
+ "active_bytes_peak_delta": 2821668352,
6
+ "after": {
7
+ "active_bytes_current": 42503244288,
8
+ "active_bytes_peak": 130563604480,
9
+ "allocated_bytes_current": 42503244288,
10
+ "allocated_bytes_peak": 127465112064,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 20763637248,
13
+ "inactive_split_bytes_peak": 46994892288,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 155463974912,
21
+ "reserved_bytes_peak": 155463974912,
22
+ "segment_count_current": 1147,
23
+ "segment_count_peak": 1147,
24
+ "timestamp_utc": "2026-08-26T19:04:05.729858Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42503244288,
28
+ "allocated_bytes_current_delta": 17960960,
29
+ "allocated_bytes_peak": 127465112064,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42485283328,
33
+ "active_bytes_peak": 127741936128,
34
+ "allocated_bytes_current": 42485283328,
35
+ "allocated_bytes_peak": 127465112064,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 18703320576,
38
+ "inactive_split_bytes_peak": 32325633536,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 132187684864,
46
+ "reserved_bytes_peak": 132187684864,
47
+ "segment_count_current": 940,
48
+ "segment_count_peak": 940,
49
+ "timestamp_utc": "2026-08-26T19:04:03.629844Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 20763637248,
55
+ "inactive_split_bytes_current_delta": 2060316672,
56
+ "inactive_split_bytes_peak": 46994892288,
57
+ "inactive_split_bytes_peak_delta": 14669258752,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 155463974912,
68
+ "reserved_bytes_current_delta": 23276290048,
69
+ "reserved_bytes_peak": 155463974912,
70
+ "reserved_bytes_peak_delta": 23276290048,
71
+ "segment_count_current": 1147,
72
+ "segment_count_current_delta": 207,
73
+ "segment_count_peak": 1147,
74
+ "segment_count_peak_delta": 207,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260826T190405730376Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:05.816618Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260826T190405730376Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2a24fc4e9476ffa47c1f1f2f9e278b6aa303a80de5d839eda5cb01d2d47d409c
3
+ size 4743045
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190406419077Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42503244288,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 130563604480,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 42503244288,
8
+ "active_bytes_peak": 130563604480,
9
+ "allocated_bytes_current": 42503244288,
10
+ "allocated_bytes_peak": 127465112064,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 20763637248,
13
+ "inactive_split_bytes_peak": 46994892288,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 155463974912,
21
+ "reserved_bytes_peak": 155463974912,
22
+ "segment_count_current": 1147,
23
+ "segment_count_peak": 1147,
24
+ "timestamp_utc": "2026-08-26T19:04:06.418670Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42503244288,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 127465112064,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42503244288,
33
+ "active_bytes_peak": 130563604480,
34
+ "allocated_bytes_current": 42503244288,
35
+ "allocated_bytes_peak": 127465112064,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 20763637248,
38
+ "inactive_split_bytes_peak": 46994892288,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 155463974912,
46
+ "reserved_bytes_peak": 155463974912,
47
+ "segment_count_current": 1147,
48
+ "segment_count_peak": 1147,
49
+ "timestamp_utc": "2026-08-26T19:04:05.827858Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 20763637248,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 46994892288,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 155463974912,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 155463974912,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1147,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1147,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260826T190406419077Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:06.429585Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260826T190406419077Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aebbfe8d86a9a148c7944cf029e67444944214d147c175352f0f429eb5660d46
3
+ size 384967
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190407024459Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42503244288,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 130563604480,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 42503244288,
8
+ "active_bytes_peak": 130563604480,
9
+ "allocated_bytes_current": 42503244288,
10
+ "allocated_bytes_peak": 127465112064,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 20763637248,
13
+ "inactive_split_bytes_peak": 46994892288,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 155463974912,
21
+ "reserved_bytes_peak": 155463974912,
22
+ "segment_count_current": 1147,
23
+ "segment_count_peak": 1147,
24
+ "timestamp_utc": "2026-08-26T19:04:07.024063Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42503244288,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 127465112064,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42503244288,
33
+ "active_bytes_peak": 130563604480,
34
+ "allocated_bytes_current": 42503244288,
35
+ "allocated_bytes_peak": 127465112064,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 20763637248,
38
+ "inactive_split_bytes_peak": 46994892288,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 155463974912,
46
+ "reserved_bytes_peak": 155463974912,
47
+ "segment_count_current": 1147,
48
+ "segment_count_peak": 1147,
49
+ "timestamp_utc": "2026-08-26T19:04:06.430686Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 20763637248,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 46994892288,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 155463974912,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 155463974912,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1147,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1147,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260826T190407024459Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:07.033945Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000005_20260826T190407024459Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fd7f049c76c68f9bcc677e416ff783ce76c3ac862ca0efeb7dc803661ca50f8d
3
+ size 384967
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190402080520Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42481551360,
3
+ "active_bytes_current_delta": 19433245696,
4
+ "active_bytes_peak": 113246329856,
5
+ "active_bytes_peak_delta": 90185441280,
6
+ "after": {
7
+ "active_bytes_current": 42481551360,
8
+ "active_bytes_peak": 113246329856,
9
+ "allocated_bytes_current": 42481551360,
10
+ "allocated_bytes_peak": 113246329856,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 18811910144,
13
+ "inactive_split_bytes_peak": 30718459904,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 117838970880,
21
+ "reserved_bytes_peak": 117838970880,
22
+ "segment_count_current": 927,
23
+ "segment_count_peak": 927,
24
+ "timestamp_utc": "2026-08-26T19:04:02.080054Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42481551360,
28
+ "allocated_bytes_current_delta": 19433245696,
29
+ "allocated_bytes_peak": 113246329856,
30
+ "allocated_bytes_peak_delta": 90185441280,
31
+ "before": {
32
+ "active_bytes_current": 23048305664,
33
+ "active_bytes_peak": 23060888576,
34
+ "allocated_bytes_current": 23048305664,
35
+ "allocated_bytes_peak": 23060888576,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 502711296,
38
+ "inactive_split_bytes_peak": 883235328,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 25784483840,
46
+ "reserved_bytes_peak": 25784483840,
47
+ "segment_count_current": 558,
48
+ "segment_count_peak": 558,
49
+ "timestamp_utc": "2026-08-26T19:03:23.745600Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18811910144,
55
+ "inactive_split_bytes_current_delta": 18309198848,
56
+ "inactive_split_bytes_peak": 30718459904,
57
+ "inactive_split_bytes_peak_delta": 29835224576,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 117838970880,
68
+ "reserved_bytes_current_delta": 92054487040,
69
+ "reserved_bytes_peak": 117838970880,
70
+ "reserved_bytes_peak_delta": 92054487040,
71
+ "segment_count_current": 927,
72
+ "segment_count_current_delta": 369,
73
+ "segment_count_peak": 927,
74
+ "segment_count_peak_delta": 369,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260826T190402080520Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:02.163072Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260826T190402080520Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a173ef4ffb81f29a415ae4991c1534810f7f11a29159ed7ba8526ced8aed8e27
3
+ size 4131448
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190402816603Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42481551360,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 127725126656,
5
+ "active_bytes_peak_delta": 14478796800,
6
+ "after": {
7
+ "active_bytes_current": 42481551360,
8
+ "active_bytes_peak": 127725126656,
9
+ "allocated_bytes_current": 42481551360,
10
+ "allocated_bytes_peak": 127448441856,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 18906281984,
13
+ "inactive_split_bytes_peak": 31017596928,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 130228944896,
21
+ "reserved_bytes_peak": 130228944896,
22
+ "segment_count_current": 935,
23
+ "segment_count_peak": 935,
24
+ "timestamp_utc": "2026-08-26T19:04:02.816197Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42481551360,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 127448441856,
30
+ "allocated_bytes_peak_delta": 14202112000,
31
+ "before": {
32
+ "active_bytes_current": 42481551360,
33
+ "active_bytes_peak": 113246329856,
34
+ "allocated_bytes_current": 42481551360,
35
+ "allocated_bytes_peak": 113246329856,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 18811910144,
38
+ "inactive_split_bytes_peak": 30718459904,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 117838970880,
46
+ "reserved_bytes_peak": 117838970880,
47
+ "segment_count_current": 927,
48
+ "segment_count_peak": 927,
49
+ "timestamp_utc": "2026-08-26T19:04:02.175428Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18906281984,
55
+ "inactive_split_bytes_current_delta": 94371840,
56
+ "inactive_split_bytes_peak": 31017596928,
57
+ "inactive_split_bytes_peak_delta": 299137024,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 130228944896,
68
+ "reserved_bytes_current_delta": 12389974016,
69
+ "reserved_bytes_peak": 130228944896,
70
+ "reserved_bytes_peak_delta": 12389974016,
71
+ "segment_count_current": 935,
72
+ "segment_count_current_delta": 8,
73
+ "segment_count_peak": 935,
74
+ "segment_count_peak_delta": 8,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260826T190402816603Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:02.893988Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260826T190402816603Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fd0e6496b8c4f5e5a893295cce203c7b318f3b559a9b814981578f0119c3b48a
3
+ size 4526765
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190403545196Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42481551360,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 127725126656,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 42481551360,
8
+ "active_bytes_peak": 127725126656,
9
+ "allocated_bytes_current": 42481551360,
10
+ "allocated_bytes_peak": 127448441856,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 18904184832,
13
+ "inactive_split_bytes_peak": 31017596928,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 130321219584,
21
+ "reserved_bytes_peak": 130321219584,
22
+ "segment_count_current": 936,
23
+ "segment_count_peak": 936,
24
+ "timestamp_utc": "2026-08-26T19:04:03.544788Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42481551360,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 127448441856,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42481551360,
33
+ "active_bytes_peak": 127725126656,
34
+ "allocated_bytes_current": 42481551360,
35
+ "allocated_bytes_peak": 127448441856,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 18906281984,
38
+ "inactive_split_bytes_peak": 31017596928,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 130228944896,
46
+ "reserved_bytes_peak": 130228944896,
47
+ "segment_count_current": 935,
48
+ "segment_count_peak": 935,
49
+ "timestamp_utc": "2026-08-26T19:04:02.904143Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 18904184832,
55
+ "inactive_split_bytes_current_delta": -2097152,
56
+ "inactive_split_bytes_peak": 31017596928,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 130321219584,
68
+ "reserved_bytes_current_delta": 92274688,
69
+ "reserved_bytes_peak": 130321219584,
70
+ "reserved_bytes_peak_delta": 92274688,
71
+ "segment_count_current": 936,
72
+ "segment_count_current_delta": 1,
73
+ "segment_count_peak": 936,
74
+ "segment_count_peak_delta": 1,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260826T190403545196Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:03.621671Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260826T190403545196Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b186e0e3517c8a4ba9a3e4dca7e14ae4fc1117cc85b8233f13e6536b0f26ff3c
3
+ size 4545022
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190405730411Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42499512320,
3
+ "active_bytes_current_delta": 17960960,
4
+ "active_bytes_peak": 130559872512,
5
+ "active_bytes_peak_delta": 2834745856,
6
+ "after": {
7
+ "active_bytes_current": 42499512320,
8
+ "active_bytes_peak": 130559872512,
9
+ "allocated_bytes_current": 42499512320,
10
+ "allocated_bytes_peak": 127448441856,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 20966598656,
13
+ "inactive_split_bytes_peak": 47197853696,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 155661107200,
21
+ "reserved_bytes_peak": 155661107200,
22
+ "segment_count_current": 1104,
23
+ "segment_count_peak": 1104,
24
+ "timestamp_utc": "2026-08-26T19:04:05.729875Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42499512320,
28
+ "allocated_bytes_current_delta": 17960960,
29
+ "allocated_bytes_peak": 127448441856,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42481551360,
33
+ "active_bytes_peak": 127725126656,
34
+ "allocated_bytes_current": 42481551360,
35
+ "allocated_bytes_peak": 127448441856,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 18904184832,
38
+ "inactive_split_bytes_peak": 31017596928,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 130321219584,
46
+ "reserved_bytes_peak": 130321219584,
47
+ "segment_count_current": 936,
48
+ "segment_count_peak": 936,
49
+ "timestamp_utc": "2026-08-26T19:04:03.631572Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 20966598656,
55
+ "inactive_split_bytes_current_delta": 2062413824,
56
+ "inactive_split_bytes_peak": 47197853696,
57
+ "inactive_split_bytes_peak_delta": 16180256768,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 155661107200,
68
+ "reserved_bytes_current_delta": 25339887616,
69
+ "reserved_bytes_peak": 155661107200,
70
+ "reserved_bytes_peak_delta": 25339887616,
71
+ "segment_count_current": 1104,
72
+ "segment_count_current_delta": 168,
73
+ "segment_count_peak": 1104,
74
+ "segment_count_peak_delta": 168,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260826T190405730411Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:05.818717Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260826T190405730411Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0e2a654dca35dfc9651a940d33d0c4f97e84e261cec94d4caed1e3bfe091e040
3
+ size 4761004
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190406419136Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42499512320,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 130559872512,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 42499512320,
8
+ "active_bytes_peak": 130559872512,
9
+ "allocated_bytes_current": 42499512320,
10
+ "allocated_bytes_peak": 127448441856,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 20966598656,
13
+ "inactive_split_bytes_peak": 47197853696,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 155661107200,
21
+ "reserved_bytes_peak": 155661107200,
22
+ "segment_count_current": 1104,
23
+ "segment_count_peak": 1104,
24
+ "timestamp_utc": "2026-08-26T19:04:06.418730Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42499512320,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 127448441856,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42499512320,
33
+ "active_bytes_peak": 130559872512,
34
+ "allocated_bytes_current": 42499512320,
35
+ "allocated_bytes_peak": 127448441856,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 20966598656,
38
+ "inactive_split_bytes_peak": 47197853696,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 155661107200,
46
+ "reserved_bytes_peak": 155661107200,
47
+ "segment_count_current": 1104,
48
+ "segment_count_peak": 1104,
49
+ "timestamp_utc": "2026-08-26T19:04:05.830255Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 20966598656,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 47197853696,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 155661107200,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 155661107200,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1104,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1104,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260826T190406419136Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:06.429430Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260826T190406419136Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:73415e2e740862df2997998353ec237668184603fe0563123d817019e4d518b7
3
+ size 402457
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190407024499Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 42499512320,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 130559872512,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 42499512320,
8
+ "active_bytes_peak": 130559872512,
9
+ "allocated_bytes_current": 42499512320,
10
+ "allocated_bytes_peak": 127448441856,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 20966598656,
13
+ "inactive_split_bytes_peak": 47197853696,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 155661107200,
21
+ "reserved_bytes_peak": 155661107200,
22
+ "segment_count_current": 1104,
23
+ "segment_count_peak": 1104,
24
+ "timestamp_utc": "2026-08-26T19:04:07.024118Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 42499512320,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 127448441856,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 42499512320,
33
+ "active_bytes_peak": 130559872512,
34
+ "allocated_bytes_current": 42499512320,
35
+ "allocated_bytes_peak": 127448441856,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 20966598656,
38
+ "inactive_split_bytes_peak": 47197853696,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 155661107200,
46
+ "reserved_bytes_peak": 155661107200,
47
+ "segment_count_current": 1104,
48
+ "segment_count_peak": 1104,
49
+ "timestamp_utc": "2026-08-26T19:04:06.430521Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 20966598656,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 47197853696,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 155661107200,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 155661107200,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1104,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1104,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260826T190407024499Z.pickle",
76
+ "timestamp_utc": "2026-08-26T19:04:07.033515Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000005_20260826T190407024499Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2a0671291f458477c03377803b0d7950ec7597b71f5fde4da7621bb7edcce385
3
+ size 402457
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/run.env ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=echo
2
+ config=DSV2_LITE_L24
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=better-memory-dump-support
6
+ commit=b7c52442
7
+ num_layers=24
8
+ hidden=2048
9
+ ffn_hidden=10944
10
+ moe_ffn_hidden=1408
11
+ num_experts=64
12
+ topk=6
13
+ experts_per_rank=8
14
+ expert_params=8650752
15
+ heads=16
16
+ kv_groups=0
17
+ kv_channels=128
18
+ qk_layernorm=1
19
+ shared_ffn=2816
20
+ shared_expert_overlap=0
21
+ mla=1
22
+ dense_first_layers=1
23
+ moe_layers=23
24
+ rotary_base=10000
25
+ rotary_scaling=40
26
+ qkv_bias=0
27
+ router_balancing=seq_aux_loss
28
+ aux_loss_coeff=1e-3
29
+ router_pre_softmax=1
30
+ router_fusion=0
31
+ router_dtype=fp32
32
+ device_initiated_grouped_gemm=1
33
+ norm_epsilon=1e-6
34
+ seq_length=8192
35
+ micro_batch_size=1
36
+ global_batch_size=8
37
+ num_microbatches=1
38
+ assignments_per_rank=49152
39
+ mean_tokens_per_expert=6144
40
+ slot_policy=matched
41
+ slots_per_rank=8
42
+ capacity_mode=static
43
+ capacity=1.25
44
+ cuda_graph=1
45
+ probe_mode=0
46
+ async_grad_reduce=0
47
+ branch_role=memory
48
+ memory_dump_mode=iteration
49
+ memory_snapshot_path=/workspace/tierb_study_20260826_144644/echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/memory_snapshots/snapshot.pickle
50
+ cuda_graph_warmup_steps=3
51
+ recompute=none
52
+ force_balance=0
53
+ train_iters=6
54
+ ep_size=8
55
+ echo_dispatch_overlap=1
56
+ echo_dispatch_recompute=0
57
+ ultraep_quota_min_tokens=
58
+ ultraep_weight_sync_join=
59
+ profile_enable=0
60
+ kernel_trace=0
61
+ fp8_recipe=mxfp8
echo/tierbmem/tierbmem_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p25__memiteration__it6/rep1/train.log ADDED
The diff for this file is too large to render. See raw diff