AbhishekGhosh1998 commited on
Commit
1f94dd4
·
verified ·
1 Parent(s): a11865c

granularity results tierb-fixed-moonep-full/moonep/tierbmem @ 2026-08-30T00:07:08Z from sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0 (part 2)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260830T000228648125Z.pickle +3 -0
  2. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260830T000229043051Z.json +79 -0
  3. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260830T000229043051Z.pickle +3 -0
  4. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260830T000229439146Z.json +79 -0
  5. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260830T000229439146Z.pickle +3 -0
  6. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260830T000227473203Z.json +79 -0
  7. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260830T000227473203Z.pickle +3 -0
  8. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260830T000228249208Z.json +79 -0
  9. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260830T000228249208Z.pickle +3 -0
  10. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260830T000228648149Z.json +79 -0
  11. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260830T000228648149Z.pickle +3 -0
  12. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260830T000229043033Z.json +79 -0
  13. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260830T000229043033Z.pickle +3 -0
  14. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260830T000229439175Z.json +79 -0
  15. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260830T000229439175Z.pickle +3 -0
  16. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260830T000227473185Z.json +79 -0
  17. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260830T000227473185Z.pickle +3 -0
  18. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260830T000228249197Z.json +79 -0
  19. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260830T000228249197Z.pickle +3 -0
  20. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260830T000228648089Z.json +79 -0
  21. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260830T000228648089Z.pickle +3 -0
  22. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260830T000229043034Z.json +79 -0
  23. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260830T000229043034Z.pickle +3 -0
  24. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260830T000229439132Z.json +79 -0
  25. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260830T000229439132Z.pickle +3 -0
  26. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260830T000227473200Z.json +79 -0
  27. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260830T000227473200Z.pickle +3 -0
  28. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260830T000228249224Z.json +79 -0
  29. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260830T000228249224Z.pickle +3 -0
  30. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260830T000228648140Z.json +79 -0
  31. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260830T000228648140Z.pickle +3 -0
  32. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260830T000229043064Z.json +79 -0
  33. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260830T000229043064Z.pickle +3 -0
  34. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260830T000229439160Z.json +79 -0
  35. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260830T000229439160Z.pickle +3 -0
  36. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/run.env +73 -0
  37. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/train.log +1302 -0
  38. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/mem_samples.csv +232 -0
  39. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260830T000343559627Z.json +79 -0
  40. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260830T000343559627Z.pickle +3 -0
  41. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260830T000344599451Z.json +79 -0
  42. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260830T000344599451Z.pickle +3 -0
  43. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260830T000345380733Z.json +79 -0
  44. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260830T000345380733Z.pickle +3 -0
  45. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260830T000347043102Z.json +79 -0
  46. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260830T000347043102Z.pickle +3 -0
  47. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260830T000347538422Z.json +79 -0
  48. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260830T000347538422Z.pickle +3 -0
  49. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260830T000347974022Z.json +79 -0
  50. tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260830T000347974022Z.pickle +3 -0
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000002_20260830T000228648125Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eee059d9f5f9fc1ec69e69a94417369ced2df3ea7f84d841c21990f1427c145f
3
+ size 1613887
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260830T000229043051Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 70442701824,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 92048031744,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 70442701824,
8
+ "active_bytes_peak": 92048031744,
9
+ "allocated_bytes_current": 70442701824,
10
+ "allocated_bytes_peak": 92048031744,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 1281993728,
13
+ "inactive_split_bytes_peak": 12227684864,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 95361695744,
21
+ "reserved_bytes_peak": 95361695744,
22
+ "segment_count_current": 1759,
23
+ "segment_count_peak": 1759,
24
+ "timestamp_utc": "2026-08-30T00:02:29.042643Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 70442701824,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 92048031744,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 70442701824,
33
+ "active_bytes_peak": 92048031744,
34
+ "allocated_bytes_current": 70442701824,
35
+ "allocated_bytes_peak": 92048031744,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 1281993728,
38
+ "inactive_split_bytes_peak": 12227684864,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 95361695744,
46
+ "reserved_bytes_peak": 95361695744,
47
+ "segment_count_current": 1759,
48
+ "segment_count_peak": 1759,
49
+ "timestamp_utc": "2026-08-30T00:02:28.679933Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1281993728,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12227684864,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 95361695744,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 95361695744,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1759,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1759,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260830T000229043051Z.pickle",
76
+ "timestamp_utc": "2026-08-30T00:02:29.073396Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000003_20260830T000229043051Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:18b67e3e4d0a110f56f9419d222d408f0eabe6117189aeb0a474f3f6fb9f15a4
3
+ size 1613889
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260830T000229439146Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 70442701824,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 92048031744,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 70442701824,
8
+ "active_bytes_peak": 92048031744,
9
+ "allocated_bytes_current": 70442701824,
10
+ "allocated_bytes_peak": 92048031744,
11
+ "device": 4,
12
+ "inactive_split_bytes_current": 1281993728,
13
+ "inactive_split_bytes_peak": 12227684864,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 4,
20
+ "reserved_bytes_current": 95361695744,
21
+ "reserved_bytes_peak": 95361695744,
22
+ "segment_count_current": 1759,
23
+ "segment_count_peak": 1759,
24
+ "timestamp_utc": "2026-08-30T00:02:29.438748Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 70442701824,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 92048031744,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 70442701824,
33
+ "active_bytes_peak": 92048031744,
34
+ "allocated_bytes_current": 70442701824,
35
+ "allocated_bytes_peak": 92048031744,
36
+ "device": 4,
37
+ "inactive_split_bytes_current": 1281993728,
38
+ "inactive_split_bytes_peak": 12227684864,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 4,
45
+ "reserved_bytes_current": 95361695744,
46
+ "reserved_bytes_peak": 95361695744,
47
+ "segment_count_current": 1759,
48
+ "segment_count_peak": 1759,
49
+ "timestamp_utc": "2026-08-30T00:02:29.076202Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 4,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1281993728,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12227684864,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 4,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 95361695744,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 95361695744,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1759,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1759,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260830T000229439146Z.pickle",
76
+ "timestamp_utc": "2026-08-30T00:02:29.469405Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0004_iter000004_20260830T000229439146Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dc1efc148313fb98501f4fa6409c287bbf447add003a1ee1180525cad94f93c5
3
+ size 1613890
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260830T000227473203Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 70438206464,
3
+ "active_bytes_current_delta": 47660868608,
4
+ "active_bytes_peak": 70438209536,
5
+ "active_bytes_peak_delta": 45747587072,
6
+ "after": {
7
+ "active_bytes_current": 70438206464,
8
+ "active_bytes_peak": 70438209536,
9
+ "allocated_bytes_current": 70438206464,
10
+ "allocated_bytes_peak": 70438209536,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 1286489088,
13
+ "inactive_split_bytes_peak": 12292982272,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 71844233216,
21
+ "reserved_bytes_peak": 71844233216,
22
+ "segment_count_current": 1692,
23
+ "segment_count_peak": 1692,
24
+ "timestamp_utc": "2026-08-30T00:02:27.472683Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 70438206464,
28
+ "allocated_bytes_current_delta": 47660868608,
29
+ "allocated_bytes_peak": 70438209536,
30
+ "allocated_bytes_peak_delta": 45747587072,
31
+ "before": {
32
+ "active_bytes_current": 22777337856,
33
+ "active_bytes_peak": 24690622464,
34
+ "allocated_bytes_current": 22777337856,
35
+ "allocated_bytes_peak": 24690622464,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 572352512,
38
+ "inactive_split_bytes_peak": 1025337344,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 33151778816,
46
+ "reserved_bytes_peak": 33151778816,
47
+ "segment_count_current": 971,
48
+ "segment_count_peak": 971,
49
+ "timestamp_utc": "2026-08-30T00:02:05.946861Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1286489088,
55
+ "inactive_split_bytes_current_delta": 714136576,
56
+ "inactive_split_bytes_peak": 12292982272,
57
+ "inactive_split_bytes_peak_delta": 11267644928,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 71844233216,
68
+ "reserved_bytes_current_delta": 38692454400,
69
+ "reserved_bytes_peak": 71844233216,
70
+ "reserved_bytes_peak_delta": 38692454400,
71
+ "segment_count_current": 1692,
72
+ "segment_count_current_delta": 721,
73
+ "segment_count_peak": 1692,
74
+ "segment_count_peak_delta": 721,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260830T000227473203Z.pickle",
76
+ "timestamp_utc": "2026-08-30T00:02:27.542532Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000000_20260830T000227473203Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bdd97f46fe7797ddadb16a6e4bc798584d406c63291e1f349c59624ac68ed81d
3
+ size 3049530
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260830T000228249208Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 70438206464,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 92041671680,
5
+ "active_bytes_peak_delta": 21603462144,
6
+ "after": {
7
+ "active_bytes_current": 70438206464,
8
+ "active_bytes_peak": 92041671680,
9
+ "allocated_bytes_current": 70438206464,
10
+ "allocated_bytes_peak": 92041671680,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 1286489088,
13
+ "inactive_split_bytes_peak": 12292982272,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 95453970432,
21
+ "reserved_bytes_peak": 95453970432,
22
+ "segment_count_current": 1746,
23
+ "segment_count_peak": 1746,
24
+ "timestamp_utc": "2026-08-30T00:02:28.248783Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 70438206464,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 92041671680,
30
+ "allocated_bytes_peak_delta": 21603462144,
31
+ "before": {
32
+ "active_bytes_current": 70438206464,
33
+ "active_bytes_peak": 70438209536,
34
+ "allocated_bytes_current": 70438206464,
35
+ "allocated_bytes_peak": 70438209536,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 1286489088,
38
+ "inactive_split_bytes_peak": 12292982272,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 71844233216,
46
+ "reserved_bytes_peak": 71844233216,
47
+ "segment_count_current": 1692,
48
+ "segment_count_peak": 1692,
49
+ "timestamp_utc": "2026-08-30T00:02:27.550926Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1286489088,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12292982272,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 95453970432,
68
+ "reserved_bytes_current_delta": 23609737216,
69
+ "reserved_bytes_peak": 95453970432,
70
+ "reserved_bytes_peak_delta": 23609737216,
71
+ "segment_count_current": 1746,
72
+ "segment_count_current_delta": 54,
73
+ "segment_count_peak": 1746,
74
+ "segment_count_peak_delta": 54,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260830T000228249208Z.pickle",
76
+ "timestamp_utc": "2026-08-30T00:02:28.282574Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000001_20260830T000228249208Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f4b62bf0af7b015f64e1095dd7dff2e6ce762d12772b980d070b29d3e78bfdf
3
+ size 1501557
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260830T000228648149Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 70438206464,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 92042135040,
5
+ "active_bytes_peak_delta": 463360,
6
+ "after": {
7
+ "active_bytes_current": 70438206464,
8
+ "active_bytes_peak": 92042135040,
9
+ "allocated_bytes_current": 70438206464,
10
+ "allocated_bytes_peak": 92042135040,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 1286489088,
13
+ "inactive_split_bytes_peak": 12292982272,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 95453970432,
21
+ "reserved_bytes_peak": 95453970432,
22
+ "segment_count_current": 1746,
23
+ "segment_count_peak": 1746,
24
+ "timestamp_utc": "2026-08-30T00:02:28.647748Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 70438206464,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 92042135040,
30
+ "allocated_bytes_peak_delta": 463360,
31
+ "before": {
32
+ "active_bytes_current": 70438206464,
33
+ "active_bytes_peak": 92041671680,
34
+ "allocated_bytes_current": 70438206464,
35
+ "allocated_bytes_peak": 92041671680,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 1286489088,
38
+ "inactive_split_bytes_peak": 12292982272,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 95453970432,
46
+ "reserved_bytes_peak": 95453970432,
47
+ "segment_count_current": 1746,
48
+ "segment_count_peak": 1746,
49
+ "timestamp_utc": "2026-08-30T00:02:28.285461Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1286489088,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12292982272,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 95453970432,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 95453970432,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1746,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1746,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260830T000228648149Z.pickle",
76
+ "timestamp_utc": "2026-08-30T00:02:28.677017Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000002_20260830T000228648149Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:222d249ff9dca1114a959342aa5f75e22f26c3f23afc04fa3e659122e192dd6a
3
+ size 1559027
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260830T000229043033Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 70438206464,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 92042135040,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 70438206464,
8
+ "active_bytes_peak": 92042135040,
9
+ "allocated_bytes_current": 70438206464,
10
+ "allocated_bytes_peak": 92042135040,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 1286489088,
13
+ "inactive_split_bytes_peak": 12292982272,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 95453970432,
21
+ "reserved_bytes_peak": 95453970432,
22
+ "segment_count_current": 1746,
23
+ "segment_count_peak": 1746,
24
+ "timestamp_utc": "2026-08-30T00:02:29.042629Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 70438206464,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 92042135040,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 70438206464,
33
+ "active_bytes_peak": 92042135040,
34
+ "allocated_bytes_current": 70438206464,
35
+ "allocated_bytes_peak": 92042135040,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 1286489088,
38
+ "inactive_split_bytes_peak": 12292982272,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 95453970432,
46
+ "reserved_bytes_peak": 95453970432,
47
+ "segment_count_current": 1746,
48
+ "segment_count_peak": 1746,
49
+ "timestamp_utc": "2026-08-30T00:02:28.679684Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1286489088,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12292982272,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 95453970432,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 95453970432,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1746,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1746,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260830T000229043033Z.pickle",
76
+ "timestamp_utc": "2026-08-30T00:02:29.073007Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000003_20260830T000229043033Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:50cd8fecc613203bfb2e5d6b6c8618a394ab018f22cdf9ed73f8e70f053a875c
3
+ size 1559121
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260830T000229439175Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 70438206464,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 92042135040,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 70438206464,
8
+ "active_bytes_peak": 92042135040,
9
+ "allocated_bytes_current": 70438206464,
10
+ "allocated_bytes_peak": 92042135040,
11
+ "device": 5,
12
+ "inactive_split_bytes_current": 1286489088,
13
+ "inactive_split_bytes_peak": 12292982272,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 5,
20
+ "reserved_bytes_current": 95453970432,
21
+ "reserved_bytes_peak": 95453970432,
22
+ "segment_count_current": 1746,
23
+ "segment_count_peak": 1746,
24
+ "timestamp_utc": "2026-08-30T00:02:29.438765Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 70438206464,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 92042135040,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 70438206464,
33
+ "active_bytes_peak": 92042135040,
34
+ "allocated_bytes_current": 70438206464,
35
+ "allocated_bytes_peak": 92042135040,
36
+ "device": 5,
37
+ "inactive_split_bytes_current": 1286489088,
38
+ "inactive_split_bytes_peak": 12292982272,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 5,
45
+ "reserved_bytes_current": 95453970432,
46
+ "reserved_bytes_peak": 95453970432,
47
+ "segment_count_current": 1746,
48
+ "segment_count_peak": 1746,
49
+ "timestamp_utc": "2026-08-30T00:02:29.075794Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 5,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1286489088,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12292982272,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 5,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 95453970432,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 95453970432,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1746,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1746,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260830T000229439175Z.pickle",
76
+ "timestamp_utc": "2026-08-30T00:02:29.469294Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0005_iter000004_20260830T000229439175Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f1aca06dbecca24b3b47c3fbfff09c4ed7cc9b5912ba1fd56338f3bb572615f8
3
+ size 1558993
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260830T000227473185Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 70438206464,
3
+ "active_bytes_current_delta": 47660868608,
4
+ "active_bytes_peak": 70438209536,
5
+ "active_bytes_peak_delta": 45747587072,
6
+ "after": {
7
+ "active_bytes_current": 70438206464,
8
+ "active_bytes_peak": 70438209536,
9
+ "allocated_bytes_current": 70438206464,
10
+ "allocated_bytes_peak": 70438209536,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 1286489088,
13
+ "inactive_split_bytes_peak": 12292982272,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 71844233216,
21
+ "reserved_bytes_peak": 71844233216,
22
+ "segment_count_current": 1692,
23
+ "segment_count_peak": 1692,
24
+ "timestamp_utc": "2026-08-30T00:02:27.472664Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 70438206464,
28
+ "allocated_bytes_current_delta": 47660868608,
29
+ "allocated_bytes_peak": 70438209536,
30
+ "allocated_bytes_peak_delta": 45747587072,
31
+ "before": {
32
+ "active_bytes_current": 22777337856,
33
+ "active_bytes_peak": 24690622464,
34
+ "allocated_bytes_current": 22777337856,
35
+ "allocated_bytes_peak": 24690622464,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 572352512,
38
+ "inactive_split_bytes_peak": 1025337344,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 33151778816,
46
+ "reserved_bytes_peak": 33151778816,
47
+ "segment_count_current": 971,
48
+ "segment_count_peak": 971,
49
+ "timestamp_utc": "2026-08-30T00:02:05.946932Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1286489088,
55
+ "inactive_split_bytes_current_delta": 714136576,
56
+ "inactive_split_bytes_peak": 12292982272,
57
+ "inactive_split_bytes_peak_delta": 11267644928,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 71844233216,
68
+ "reserved_bytes_current_delta": 38692454400,
69
+ "reserved_bytes_peak": 71844233216,
70
+ "reserved_bytes_peak_delta": 38692454400,
71
+ "segment_count_current": 1692,
72
+ "segment_count_current_delta": 721,
73
+ "segment_count_peak": 1692,
74
+ "segment_count_peak_delta": 721,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260830T000227473185Z.pickle",
76
+ "timestamp_utc": "2026-08-30T00:02:27.540528Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000000_20260830T000227473185Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e6e55753131b4e2261bba94d80b2924d6650dc796b7700adff66c77c14c4cae8
3
+ size 3049530
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260830T000228249197Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 70438206464,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 92041671680,
5
+ "active_bytes_peak_delta": 21603462144,
6
+ "after": {
7
+ "active_bytes_current": 70438206464,
8
+ "active_bytes_peak": 92041671680,
9
+ "allocated_bytes_current": 70438206464,
10
+ "allocated_bytes_peak": 92041671680,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 1286489088,
13
+ "inactive_split_bytes_peak": 12292982272,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 95453970432,
21
+ "reserved_bytes_peak": 95453970432,
22
+ "segment_count_current": 1746,
23
+ "segment_count_peak": 1746,
24
+ "timestamp_utc": "2026-08-30T00:02:28.248775Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 70438206464,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 92041671680,
30
+ "allocated_bytes_peak_delta": 21603462144,
31
+ "before": {
32
+ "active_bytes_current": 70438206464,
33
+ "active_bytes_peak": 70438209536,
34
+ "allocated_bytes_current": 70438206464,
35
+ "allocated_bytes_peak": 70438209536,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 1286489088,
38
+ "inactive_split_bytes_peak": 12292982272,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 71844233216,
46
+ "reserved_bytes_peak": 71844233216,
47
+ "segment_count_current": 1692,
48
+ "segment_count_peak": 1692,
49
+ "timestamp_utc": "2026-08-30T00:02:27.548623Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1286489088,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12292982272,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 95453970432,
68
+ "reserved_bytes_current_delta": 23609737216,
69
+ "reserved_bytes_peak": 95453970432,
70
+ "reserved_bytes_peak_delta": 23609737216,
71
+ "segment_count_current": 1746,
72
+ "segment_count_current_delta": 54,
73
+ "segment_count_peak": 1746,
74
+ "segment_count_peak_delta": 54,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260830T000228249197Z.pickle",
76
+ "timestamp_utc": "2026-08-30T00:02:28.280749Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000001_20260830T000228249197Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:efa26c1491f2d6c2946f387b0ee0760dbd35990fdf8bdbd05044093f07be4b6b
3
+ size 1501559
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260830T000228648089Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 70438206464,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 92042135040,
5
+ "active_bytes_peak_delta": 463360,
6
+ "after": {
7
+ "active_bytes_current": 70438206464,
8
+ "active_bytes_peak": 92042135040,
9
+ "allocated_bytes_current": 70438206464,
10
+ "allocated_bytes_peak": 92042135040,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 1286489088,
13
+ "inactive_split_bytes_peak": 12292982272,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 95453970432,
21
+ "reserved_bytes_peak": 95453970432,
22
+ "segment_count_current": 1746,
23
+ "segment_count_peak": 1746,
24
+ "timestamp_utc": "2026-08-30T00:02:28.647693Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 70438206464,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 92042135040,
30
+ "allocated_bytes_peak_delta": 463360,
31
+ "before": {
32
+ "active_bytes_current": 70438206464,
33
+ "active_bytes_peak": 92041671680,
34
+ "allocated_bytes_current": 70438206464,
35
+ "allocated_bytes_peak": 92041671680,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 1286489088,
38
+ "inactive_split_bytes_peak": 12292982272,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 95453970432,
46
+ "reserved_bytes_peak": 95453970432,
47
+ "segment_count_current": 1746,
48
+ "segment_count_peak": 1746,
49
+ "timestamp_utc": "2026-08-30T00:02:28.283570Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1286489088,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12292982272,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 95453970432,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 95453970432,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1746,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1746,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260830T000228648089Z.pickle",
76
+ "timestamp_utc": "2026-08-30T00:02:28.675437Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000002_20260830T000228648089Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d98287d71e523a7256595ec4409fa8ea1e389d69e510f64cc9f62e62fa581173
3
+ size 1559029
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260830T000229043034Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 70438206464,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 92042135040,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 70438206464,
8
+ "active_bytes_peak": 92042135040,
9
+ "allocated_bytes_current": 70438206464,
10
+ "allocated_bytes_peak": 92042135040,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 1286489088,
13
+ "inactive_split_bytes_peak": 12292982272,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 95453970432,
21
+ "reserved_bytes_peak": 95453970432,
22
+ "segment_count_current": 1746,
23
+ "segment_count_peak": 1746,
24
+ "timestamp_utc": "2026-08-30T00:02:29.042625Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 70438206464,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 92042135040,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 70438206464,
33
+ "active_bytes_peak": 92042135040,
34
+ "allocated_bytes_current": 70438206464,
35
+ "allocated_bytes_peak": 92042135040,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 1286489088,
38
+ "inactive_split_bytes_peak": 12292982272,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 95453970432,
46
+ "reserved_bytes_peak": 95453970432,
47
+ "segment_count_current": 1746,
48
+ "segment_count_peak": 1746,
49
+ "timestamp_utc": "2026-08-30T00:02:28.678050Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1286489088,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12292982272,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 95453970432,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 95453970432,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1746,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1746,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260830T000229043034Z.pickle",
76
+ "timestamp_utc": "2026-08-30T00:02:29.072404Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000003_20260830T000229043034Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3141c1a980654643dabaf2fbf19a5f86d2d2efecd4e0190d5f5ef481d3cb2029
3
+ size 1559123
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260830T000229439132Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 70438206464,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 92042135040,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 70438206464,
8
+ "active_bytes_peak": 92042135040,
9
+ "allocated_bytes_current": 70438206464,
10
+ "allocated_bytes_peak": 92042135040,
11
+ "device": 6,
12
+ "inactive_split_bytes_current": 1286489088,
13
+ "inactive_split_bytes_peak": 12292982272,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 6,
20
+ "reserved_bytes_current": 95453970432,
21
+ "reserved_bytes_peak": 95453970432,
22
+ "segment_count_current": 1746,
23
+ "segment_count_peak": 1746,
24
+ "timestamp_utc": "2026-08-30T00:02:29.438735Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 70438206464,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 92042135040,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 70438206464,
33
+ "active_bytes_peak": 92042135040,
34
+ "allocated_bytes_current": 70438206464,
35
+ "allocated_bytes_peak": 92042135040,
36
+ "device": 6,
37
+ "inactive_split_bytes_current": 1286489088,
38
+ "inactive_split_bytes_peak": 12292982272,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 6,
45
+ "reserved_bytes_current": 95453970432,
46
+ "reserved_bytes_peak": 95453970432,
47
+ "segment_count_current": 1746,
48
+ "segment_count_peak": 1746,
49
+ "timestamp_utc": "2026-08-30T00:02:29.075223Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 6,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1286489088,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12292982272,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 6,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 95453970432,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 95453970432,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1746,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1746,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260830T000229439132Z.pickle",
76
+ "timestamp_utc": "2026-08-30T00:02:29.467539Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0006_iter000004_20260830T000229439132Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b082901926d40f821fbc668237652c19fa42000b2f9f32a0044f1cd0bfaaf9dd
3
+ size 1558995
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260830T000227473200Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 70438206464,
3
+ "active_bytes_current_delta": 47660868608,
4
+ "active_bytes_peak": 70438209536,
5
+ "active_bytes_peak_delta": 45747587072,
6
+ "after": {
7
+ "active_bytes_current": 70438206464,
8
+ "active_bytes_peak": 70438209536,
9
+ "allocated_bytes_current": 70438206464,
10
+ "allocated_bytes_peak": 70438209536,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 1286489088,
13
+ "inactive_split_bytes_peak": 12292982272,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 71844233216,
21
+ "reserved_bytes_peak": 71844233216,
22
+ "segment_count_current": 1692,
23
+ "segment_count_peak": 1692,
24
+ "timestamp_utc": "2026-08-30T00:02:27.472668Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 70438206464,
28
+ "allocated_bytes_current_delta": 47660868608,
29
+ "allocated_bytes_peak": 70438209536,
30
+ "allocated_bytes_peak_delta": 45747587072,
31
+ "before": {
32
+ "active_bytes_current": 22777337856,
33
+ "active_bytes_peak": 24690622464,
34
+ "allocated_bytes_current": 22777337856,
35
+ "allocated_bytes_peak": 24690622464,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 572352512,
38
+ "inactive_split_bytes_peak": 1025337344,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 33151778816,
46
+ "reserved_bytes_peak": 33151778816,
47
+ "segment_count_current": 971,
48
+ "segment_count_peak": 971,
49
+ "timestamp_utc": "2026-08-30T00:02:05.946870Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1286489088,
55
+ "inactive_split_bytes_current_delta": 714136576,
56
+ "inactive_split_bytes_peak": 12292982272,
57
+ "inactive_split_bytes_peak_delta": 11267644928,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 71844233216,
68
+ "reserved_bytes_current_delta": 38692454400,
69
+ "reserved_bytes_peak": 71844233216,
70
+ "reserved_bytes_peak_delta": 38692454400,
71
+ "segment_count_current": 1692,
72
+ "segment_count_current_delta": 721,
73
+ "segment_count_peak": 1692,
74
+ "segment_count_peak_delta": 721,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260830T000227473200Z.pickle",
76
+ "timestamp_utc": "2026-08-30T00:02:27.540421Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000000_20260830T000227473200Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1986a43bc8249b4db6cb6330f5e4db96409ba47e367ce58a3f2d1ac3be0698cb
3
+ size 3049528
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260830T000228249224Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 70438206464,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 92041671680,
5
+ "active_bytes_peak_delta": 21603462144,
6
+ "after": {
7
+ "active_bytes_current": 70438206464,
8
+ "active_bytes_peak": 92041671680,
9
+ "allocated_bytes_current": 70438206464,
10
+ "allocated_bytes_peak": 92041671680,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 1286489088,
13
+ "inactive_split_bytes_peak": 12292982272,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 95453970432,
21
+ "reserved_bytes_peak": 95453970432,
22
+ "segment_count_current": 1746,
23
+ "segment_count_peak": 1746,
24
+ "timestamp_utc": "2026-08-30T00:02:28.248815Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 70438206464,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 92041671680,
30
+ "allocated_bytes_peak_delta": 21603462144,
31
+ "before": {
32
+ "active_bytes_current": 70438206464,
33
+ "active_bytes_peak": 70438209536,
34
+ "allocated_bytes_current": 70438206464,
35
+ "allocated_bytes_peak": 70438209536,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 1286489088,
38
+ "inactive_split_bytes_peak": 12292982272,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 71844233216,
46
+ "reserved_bytes_peak": 71844233216,
47
+ "segment_count_current": 1692,
48
+ "segment_count_peak": 1692,
49
+ "timestamp_utc": "2026-08-30T00:02:27.548024Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1286489088,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12292982272,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 95453970432,
68
+ "reserved_bytes_current_delta": 23609737216,
69
+ "reserved_bytes_peak": 95453970432,
70
+ "reserved_bytes_peak_delta": 23609737216,
71
+ "segment_count_current": 1746,
72
+ "segment_count_current_delta": 54,
73
+ "segment_count_peak": 1746,
74
+ "segment_count_peak_delta": 54,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260830T000228249224Z.pickle",
76
+ "timestamp_utc": "2026-08-30T00:02:28.281430Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000001_20260830T000228249224Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:32a1c028300f7f7a4a0557231c30edaaaf0b48a811ad25221458a9b5bf16b285
3
+ size 1501559
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260830T000228648140Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 70438206464,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 92042135040,
5
+ "active_bytes_peak_delta": 463360,
6
+ "after": {
7
+ "active_bytes_current": 70438206464,
8
+ "active_bytes_peak": 92042135040,
9
+ "allocated_bytes_current": 70438206464,
10
+ "allocated_bytes_peak": 92042135040,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 1286489088,
13
+ "inactive_split_bytes_peak": 12292982272,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 95453970432,
21
+ "reserved_bytes_peak": 95453970432,
22
+ "segment_count_current": 1746,
23
+ "segment_count_peak": 1746,
24
+ "timestamp_utc": "2026-08-30T00:02:28.647750Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 70438206464,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 92042135040,
30
+ "allocated_bytes_peak_delta": 463360,
31
+ "before": {
32
+ "active_bytes_current": 70438206464,
33
+ "active_bytes_peak": 92041671680,
34
+ "allocated_bytes_current": 70438206464,
35
+ "allocated_bytes_peak": 92041671680,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 1286489088,
38
+ "inactive_split_bytes_peak": 12292982272,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 95453970432,
46
+ "reserved_bytes_peak": 95453970432,
47
+ "segment_count_current": 1746,
48
+ "segment_count_peak": 1746,
49
+ "timestamp_utc": "2026-08-30T00:02:28.284151Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1286489088,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12292982272,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 95453970432,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 95453970432,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1746,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1746,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260830T000228648140Z.pickle",
76
+ "timestamp_utc": "2026-08-30T00:02:28.674999Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000002_20260830T000228648140Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f7f68728a88841660ed74a69057541e9d9336833dc5c428452b5c05082ba9eff
3
+ size 1559029
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260830T000229043064Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 70438206464,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 92042135040,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 70438206464,
8
+ "active_bytes_peak": 92042135040,
9
+ "allocated_bytes_current": 70438206464,
10
+ "allocated_bytes_peak": 92042135040,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 1286489088,
13
+ "inactive_split_bytes_peak": 12292982272,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 95453970432,
21
+ "reserved_bytes_peak": 95453970432,
22
+ "segment_count_current": 1746,
23
+ "segment_count_peak": 1746,
24
+ "timestamp_utc": "2026-08-30T00:02:29.042673Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 70438206464,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 92042135040,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 70438206464,
33
+ "active_bytes_peak": 92042135040,
34
+ "allocated_bytes_current": 70438206464,
35
+ "allocated_bytes_peak": 92042135040,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 1286489088,
38
+ "inactive_split_bytes_peak": 12292982272,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 95453970432,
46
+ "reserved_bytes_peak": 95453970432,
47
+ "segment_count_current": 1746,
48
+ "segment_count_peak": 1746,
49
+ "timestamp_utc": "2026-08-30T00:02:28.677552Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1286489088,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12292982272,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 95453970432,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 95453970432,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1746,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1746,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260830T000229043064Z.pickle",
76
+ "timestamp_utc": "2026-08-30T00:02:29.069963Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000003_20260830T000229043064Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f07e1709f057110e692b7c444caed0d17bb4617d5bb137e9f17f3cddc0f4cd7a
3
+ size 1559123
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260830T000229439160Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 70438206464,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 92042135040,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 70438206464,
8
+ "active_bytes_peak": 92042135040,
9
+ "allocated_bytes_current": 70438206464,
10
+ "allocated_bytes_peak": 92042135040,
11
+ "device": 7,
12
+ "inactive_split_bytes_current": 1286489088,
13
+ "inactive_split_bytes_peak": 12292982272,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 7,
20
+ "reserved_bytes_current": 95453970432,
21
+ "reserved_bytes_peak": 95453970432,
22
+ "segment_count_current": 1746,
23
+ "segment_count_peak": 1746,
24
+ "timestamp_utc": "2026-08-30T00:02:29.438777Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 70438206464,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 92042135040,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 70438206464,
33
+ "active_bytes_peak": 92042135040,
34
+ "allocated_bytes_current": 70438206464,
35
+ "allocated_bytes_peak": 92042135040,
36
+ "device": 7,
37
+ "inactive_split_bytes_current": 1286489088,
38
+ "inactive_split_bytes_peak": 12292982272,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 7,
45
+ "reserved_bytes_current": 95453970432,
46
+ "reserved_bytes_peak": 95453970432,
47
+ "segment_count_current": 1746,
48
+ "segment_count_peak": 1746,
49
+ "timestamp_utc": "2026-08-30T00:02:29.072580Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 7,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1286489088,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 12292982272,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 7,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 95453970432,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 95453970432,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 1746,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 1746,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260830T000229439160Z.pickle",
76
+ "timestamp_utc": "2026-08-30T00:02:29.467379Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot_iteration_rank0007_iter000004_20260830T000229439160Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:73a8ba81c286e50a925efdafe625870ca009592535941a1680e19ee2259ae2b3
3
+ size 1558995
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/run.env ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=QWEN3_235B
3
+ family=tierB
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=moonep_backend_async_grad_reduce_better_memory_dump_support
6
+ commit=51c387c7
7
+ moonep_source_dir=/workspace/MoonEP
8
+ moonep_source_branch=megatron-ddp-symmetric-params
9
+ moonep_source_commit=d8bc666
10
+ num_layers=12
11
+ hidden=4096
12
+ ffn_hidden=12288
13
+ moe_ffn_hidden=1536
14
+ num_experts=128
15
+ topk=8
16
+ experts_per_rank=16
17
+ expert_params=18874368
18
+ heads=64
19
+ kv_groups=4
20
+ kv_channels=128
21
+ qk_layernorm=1
22
+ shared_ffn=0
23
+ shared_expert_overlap=0
24
+ moe_token_dispatcher=flex
25
+ mla=0
26
+ dense_first_layers=0
27
+ moe_layers=12
28
+ rotary_base=1000000
29
+ rotary_scaling=
30
+ qkv_bias=0
31
+ router_balancing=aux_loss
32
+ aux_loss_coeff=1e-3
33
+ router_pre_softmax=0
34
+ router_fusion=0
35
+ router_dtype=fp32
36
+ device_initiated_grouped_gemm=1
37
+ norm_epsilon=1e-6
38
+ seq_length=8192
39
+ micro_batch_size=1
40
+ global_batch_size=8
41
+ num_microbatches=1
42
+ assignments_per_rank=65536
43
+ mean_tokens_per_expert=4096
44
+ slot_policy=matched
45
+ slots_per_rank=16
46
+ capacity_mode=none
47
+ capacity=none
48
+ cuda_graph=0
49
+ probe_mode=0
50
+ async_grad_reduce=1
51
+ branch_role=memory
52
+ memory_dump_mode=iteration
53
+ memory_snapshot_path=/workspace/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot.pickle
54
+ cuda_graph_warmup_steps=3
55
+ recompute=none
56
+ force_balance=0
57
+ train_iters=5
58
+ ep_size=8
59
+ echo_dispatch_overlap=1
60
+ echo_dispatch_recompute=0
61
+ ultraep_quota_min_tokens=
62
+ ultraep_weight_sync_join=
63
+ profile_enable=0
64
+ kernel_trace=0
65
+ fp8_mode=mxfp8
66
+ fp8_recipe=mxfp8
67
+ training_seed=1234
68
+ learning_rate=1e-4
69
+ min_learning_rate=1.0e-5
70
+ lr_warmup_iters=5
71
+ state_dump_enable=0
72
+ state_dump_dir=
73
+ moonep_assert_no_pending_reduce=1
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/train.log ADDED
@@ -0,0 +1,1302 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [00:01:36] WARN: moonep_backend_async_grad_reduce_better_memory_dump_support lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical)
2
+ ==============================================================
3
+ QWEN3_235B / moonep (slots/rank=16 policy=matched)
4
+ E=128 K=8 H'=1536 m=16
5
+ L=12 seq=8192 gbs=8 (1 microbatches)
6
+ cap_mode=none cap=none cg=0 recompute=none
7
+ memory_dump=iteration snapshots=/workspace/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot.pickle
8
+ repo=/workspace/Megatron-MoE-Echo @ 51c387c7
9
+ ==============================================================
10
+ W0830 00:01:38.236000 271073 torch/distributed/run.py:766]
11
+ W0830 00:01:38.236000 271073 torch/distributed/run.py:766] *****************************************
12
+ W0830 00:01:38.236000 271073 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
13
+ W0830 00:01:38.236000 271073 torch/distributed/run.py:766] *****************************************
14
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
15
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
16
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
17
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
18
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
19
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
20
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
21
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
22
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
23
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
24
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
25
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
26
+ WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it
27
+ using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1
28
+ WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer
29
+ Number of virtual stages per pipeline stage: None
30
+ accumulate and all-reduce gradients in fp32 for bfloat16 data type.
31
+ using torch.bfloat16 for parameters ...
32
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
33
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
34
+ ------------------------ arguments ------------------------
35
+ account_for_embedding_in_pipeline_split ......... False
36
+ account_for_loss_in_pipeline_split .............. False
37
+ accumulate_allreduce_grads_in_fp32 .............. True
38
+ adam_beta1 ...................................... 0.9
39
+ adam_beta2 ...................................... 0.999
40
+ adam_eps ........................................ 1e-08
41
+ add_bias_linear ................................. False
42
+ add_position_embedding .......................... False
43
+ add_qkv_bias .................................... False
44
+ adlr_autoresume ................................. False
45
+ adlr_autoresume_interval ........................ 1000
46
+ align_grad_reduce ............................... True
47
+ align_param_gather .............................. False
48
+ app_tag_run_name ................................ None
49
+ app_tag_run_version ............................. 0.0.0
50
+ apply_layernorm_1p .............................. False
51
+ apply_query_key_layer_scaling ................... False
52
+ apply_residual_connection_post_layernorm ........ False
53
+ apply_rope_fusion ............................... True
54
+ async_save ...................................... None
55
+ async_tensor_model_parallel_allreduce ........... True
56
+ attention_backend ............................... AttnBackend.auto
57
+ attention_dropout ............................... 0.0
58
+ attention_softmax_in_fp32 ....................... False
59
+ auto_detect_ckpt_format ......................... False
60
+ barrier_with_L1_time ............................ True
61
+ bert_binary_head ................................ True
62
+ bert_embedder_type .............................. megatron
63
+ bert_load ....................................... None
64
+ bf16 ............................................ True
65
+ bias_dropout_fusion ............................. True
66
+ bias_gelu_fusion ................................ False
67
+ bias_swiglu_fusion .............................. True
68
+ biencoder_projection_dim ........................ 0
69
+ biencoder_shared_query_context_model ............ False
70
+ block_data_path ................................. None
71
+ calc_ft_timeouts ................................ False
72
+ calculate_per_token_loss ........................ False
73
+ check_for_large_grads ........................... False
74
+ check_for_nan_in_loss_and_grad .................. True
75
+ check_for_spiky_loss ............................ False
76
+ check_weight_hash_across_dp_replicas_interval ... None
77
+ ckpt_assume_constant_structure .................. False
78
+ ckpt_convert_format ............................. None
79
+ ckpt_convert_save ............................... None
80
+ ckpt_convert_update_legacy_dist_opt_format ...... False
81
+ ckpt_format ..................................... torch_dist
82
+ ckpt_fully_parallel_load ........................ False
83
+ ckpt_fully_parallel_save ........................ True
84
+ ckpt_fully_parallel_save_deprecated ............. False
85
+ ckpt_step ....................................... None
86
+ classes_fraction ................................ 1.0
87
+ clip_grad ....................................... 1.0
88
+ clone_scatter_output_in_embedding ............... True
89
+ config_logger_dir ...............................
90
+ consumed_train_samples .......................... 0
91
+ consumed_valid_samples .......................... 0
92
+ context_parallel_size ........................... 1
93
+ cp_comm_type .................................... ['p2p']
94
+ create_attention_mask_in_dataloader ............. True
95
+ cross_entropy_fusion_impl ....................... native
96
+ cross_entropy_loss_fusion ....................... False
97
+ cuda_graph_scope ................................ full
98
+ cuda_graph_warmup_steps ......................... 3
99
+ data_args_path .................................. None
100
+ data_cache_path ................................. None
101
+ data_parallel_random_init ....................... False
102
+ data_parallel_sharding_strategy ................. no_shard
103
+ data_parallel_size .............................. 8
104
+ data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document']
105
+ data_per_class_fraction ......................... 1.0
106
+ data_sharding ................................... True
107
+ dataloader_type ................................. single
108
+ ddp_average_in_collective ....................... False
109
+ ddp_bucket_size ................................. None
110
+ ddp_num_buckets ................................. None
111
+ ddp_pad_buckets_for_high_nccl_busbw ............. False
112
+ decoder_first_pipeline_num_layers ............... None
113
+ decoder_last_pipeline_num_layers ................ None
114
+ decoder_num_layers .............................. None
115
+ decoder_seq_length .............................. None
116
+ decoupled_lr .................................... None
117
+ decoupled_min_lr ................................ None
118
+ decrease_batch_size_if_needed ................... False
119
+ defer_embedding_wgrad_compute ................... False
120
+ delay_wgrad_compute ............................. False
121
+ deprecated_use_mcore_models ..................... True
122
+ deterministic_mode .............................. False
123
+ dino_bottleneck_size ............................ 256
124
+ dino_freeze_last_layer .......................... 1
125
+ dino_head_hidden_size ........................... 2048
126
+ dino_local_crops_number ......................... 10
127
+ dino_local_img_size ............................. 96
128
+ dino_norm_last_layer ............................ False
129
+ dino_teacher_temp ............................... 0.07
130
+ dino_warmup_teacher_temp ........................ 0.04
131
+ dino_warmup_teacher_temp_epochs ................. 30
132
+ disable_bf16_reduced_precision_matmul ........... False
133
+ disable_mamba_mem_eff_path ...................... False
134
+ disable_straggler_on_startup .................... False
135
+ dist_ckpt_format_deprecated ..................... None
136
+ dist_ckpt_strictness ............................ assume_ok_unexpected
137
+ distribute_saved_activations .................... False
138
+ distributed_backend ............................. nccl
139
+ distributed_timeout_minutes ..................... 10
140
+ embedding_init_method_std ....................... None
141
+ embedding_path .................................. None
142
+ empty_unused_memory_level ....................... 0
143
+ enable_cuda_graph ............................... False
144
+ enable_experimental ............................. False
145
+ enable_ft_package ............................... False
146
+ enable_gloo_process_groups ...................... True
147
+ enable_msc ...................................... True
148
+ enable_one_logger ............................... True
149
+ encoder_num_layers .............................. 12
150
+ encoder_seq_length .............................. 8192
151
+ end_weight_decay ................................ 0.1
152
+ eod_mask_loss ................................... False
153
+ error_injection_rate ............................ 0
154
+ error_injection_type ............................ transient_error
155
+ eval_interval ................................... 1000000
156
+ eval_iters ...................................... 0
157
+ evidence_data_path .............................. None
158
+ exit_duration_in_mins ........................... None
159
+ exit_interval ................................... None
160
+ exit_on_missing_checkpoint ...................... False
161
+ exit_signal_handler ............................. False
162
+ exp_avg_dtype ................................... torch.float32
163
+ exp_avg_sq_dtype ................................ torch.float32
164
+ expert_model_parallel_size ...................... 8
165
+ expert_tensor_parallel_size ..................... 1
166
+ export_force_local_attention .................... False
167
+ export_kd_cfg ................................... None
168
+ export_kd_teacher_ckpt_format ................... None
169
+ export_kd_teacher_load .......................... None
170
+ export_kv_cache_quant ........................... False
171
+ export_legacy_megatron .......................... False
172
+ export_model_type ............................... GPTModel
173
+ export_moe_apply_probs_on_input ................. False
174
+ export_qk_l2_norm ............................... False
175
+ export_quant_cfg ................................ None
176
+ export_real_quant_cfg ........................... None
177
+ export_te_mcore_model ........................... False
178
+ external_cuda_graph ............................. False
179
+ ffn_hidden_size ................................. 12288
180
+ finetune ........................................ False
181
+ finetune_data_split ............................. train
182
+ finetune_hf_dataset ............................. None
183
+ first_last_layers_bf16 .......................... False
184
+ flash_decode .................................... False
185
+ fp16 ............................................ False
186
+ fp16_lm_cross_entropy ........................... False
187
+ fp32_residual_connection ........................ False
188
+ fp8 ............................................. e4m3
189
+ fp8_amax_compute_algo ........................... most_recent
190
+ fp8_amax_history_len ............................ 1
191
+ fp8_interval .................................... 1
192
+ fp8_margin ...................................... 0
193
+ fp8_param_gather ................................ False
194
+ fp8_recipe ...................................... mxfp8
195
+ fp8_wgrad ....................................... True
196
+ fsdp_double_buffer .............................. False
197
+ global_batch_size ............................... 8
198
+ grad_reduce_in_bf16 ............................. False
199
+ gradient_accumulation_fusion .................... True
200
+ gradient_reduce_div_fusion ...................... True
201
+ group_query_attention ........................... True
202
+ head_lr_mult .................................... 1.0
203
+ heterogeneous_layers_config_encoded_json ........ None
204
+ heterogeneous_layers_config_path ................ None
205
+ hidden_dropout .................................. 0.0
206
+ hidden_size ..................................... 4096
207
+ hierarchical_context_parallel_sizes ............. None
208
+ high_priority_stream_groups ..................... []
209
+ hybrid_attention_ratio .......................... 0.0
210
+ hybrid_mlp_ratio ................................ 0.0
211
+ hybrid_override_pattern ......................... None
212
+ hysteresis ...................................... 2
213
+ ict_head_size ................................... None
214
+ ict_load ........................................ None
215
+ img_h ........................................... 224
216
+ img_w ........................................... 224
217
+ indexer_batch_size .............................. 128
218
+ indexer_log_interval ............................ 1000
219
+ inference_batch_times_seqlen_threshold .......... -1
220
+ inference_dynamic_batching ...................... False
221
+ inference_dynamic_batching_buffer_guaranteed_fraction 0.2
222
+ inference_dynamic_batching_buffer_overflow_factor None
223
+ inference_dynamic_batching_buffer_size_gb ....... 40.0
224
+ inference_dynamic_batching_chunk_size ........... 256
225
+ inference_dynamic_batching_max_requests_override None
226
+ inference_dynamic_batching_max_tokens_override .. None
227
+ inference_dynamic_batching_num_cuda_graphs ...... 16
228
+ inference_max_batch_size ........................ 8
229
+ inference_max_seq_length ........................ 2560
230
+ inference_rng_tracker ........................... False
231
+ init_method_std ................................. 0.01
232
+ init_method_xavier_uniform ...................... False
233
+ init_model_with_meta_device ..................... False
234
+ initial_loss_scale .............................. 4294967296
235
+ inprocess_active_world_size ..................... 8
236
+ inprocess_barrier_timeout ....................... 120
237
+ inprocess_completion_timeout .................... 120
238
+ inprocess_empty_cuda_cache ...................... False
239
+ inprocess_granularity ........................... node
240
+ inprocess_hard_timeout .......................... 90
241
+ inprocess_heartbeat_interval .................... 30
242
+ inprocess_heartbeat_timeout ..................... 60
243
+ inprocess_last_call_wait ........................ 1
244
+ inprocess_max_iterations ........................ None
245
+ inprocess_monitor_process_interval .............. 1.0
246
+ inprocess_monitor_thread_interval ............... 1.0
247
+ inprocess_progress_watchdog_interval ............ 1.0
248
+ inprocess_restart ............................... False
249
+ inprocess_soft_timeout .......................... 60
250
+ inprocess_termination_grace_time ................ 1
251
+ is_hybrid_model ................................. False
252
+ iter_per_epoch .................................. 1250
253
+ iterations_to_skip .............................. []
254
+ keep_fp8_transpose_cache_when_using_custom_fsdp . False
255
+ kitchen_config_file ............................. None
256
+ kitchen_recipe_number ........................... None
257
+ kv_channels ..................................... 128
258
+ kv_lora_rank .................................... 32
259
+ lazy_mpu_init ................................... None
260
+ load ............................................ None
261
+ load_main_params_from_ckpt ...................... None
262
+ load_model_opt_format ........................... False
263
+ local_rank ...................................... 0
264
+ log_energy ...................................... False
265
+ log_interval .................................... 1
266
+ log_loss_scale_to_tensorboard ................... True
267
+ log_memory_to_tensorboard ....................... False
268
+ log_num_zeros_in_grad ........................... False
269
+ log_params_norm ................................. False
270
+ log_progress .................................... False
271
+ log_straggler ................................... False
272
+ log_throughput .................................. True
273
+ log_timers_to_tensorboard ....................... False
274
+ log_validation_ppl_to_tensorboard ............... False
275
+ log_world_size_to_tensorboard ................... False
276
+ logging_level ................................... None
277
+ loss_scale ...................................... None
278
+ loss_scale_window ............................... 1000
279
+ lr .............................................. 0.0001
280
+ lr_decay_iters .................................. 8
281
+ lr_decay_samples ................................ None
282
+ lr_decay_style .................................. cosine
283
+ lr_warmup_fraction .............................. None
284
+ lr_warmup_init .................................. 0.0
285
+ lr_warmup_iters ................................. 5
286
+ lr_warmup_samples ............................... 0
287
+ lr_wsd_decay_iters .............................. None
288
+ lr_wsd_decay_samples ............................ None
289
+ lr_wsd_decay_style .............................. exponential
290
+ main_grads_dtype ................................ torch.float32
291
+ main_params_dtype ............................... torch.float32
292
+ make_vocab_size_divisible_by .................... 128
293
+ mamba_head_dim .................................. 64
294
+ mamba_num_groups ................................ 8
295
+ mamba_num_heads ................................. None
296
+ mamba_state_dim ................................. 128
297
+ manual_gc ....................................... False
298
+ manual_gc_eval .................................. True
299
+ manual_gc_interval .............................. 0
300
+ mask_factor ..................................... 1.0
301
+ mask_prob ....................................... 0.15
302
+ mask_type ....................................... random
303
+ masked_softmax_fusion ........................... False
304
+ max_position_embeddings ......................... 32768
305
+ max_tokens_to_oom ............................... 12000
306
+ memory_snapshot_path ............................ /workspace/tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__nocg__async__memiteration__it5/rep1/memory_snapshots/snapshot.pickle
307
+ merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt
308
+ micro_batch_size ................................ 1
309
+ microbatch_group_size_per_vp_stage .............. None
310
+ mid_level_dataset_surplus ....................... 0.005
311
+ min_loss_scale .................................. 1.0
312
+ min_lr .......................................... 1e-05
313
+ mlp_chunks_for_prefill .......................... 1
314
+ mmap_bin_files .................................. True
315
+ mock_data ....................................... False
316
+ moe_apply_probs_on_input ........................ False
317
+ moe_aux_loss_coeff .............................. 0.001
318
+ moe_deepep_num_sms .............................. 20
319
+ moe_echo_enable_random_offloading ............... False
320
+ moe_echo_expert_dispatch_overlap ................ False
321
+ moe_echo_expert_dispatcher_type ................. hybridep
322
+ moe_echo_recompute_expert_dispatch .............. False
323
+ moe_enable_deepep ............................... False
324
+ moe_enable_echo ................................. False
325
+ moe_enable_hybridep ............................. False
326
+ moe_enable_moonep ............................... True
327
+ moe_expert_capacity_factor ...................... None
328
+ moe_extended_tp ................................. False
329
+ moe_ffn_hidden_size ............................. 1536
330
+ moe_grouped_gemm ................................ True
331
+ moe_hybridep_num_sms ............................ 16
332
+ moe_input_jitter_eps ............................ None
333
+ moe_layer_freq .................................. 1
334
+ moe_layer_recompute ............................. False
335
+ moe_moonep_async_grad_reduce .................... True
336
+ moe_moonep_num_sms .............................. None
337
+ moe_num_echo_experts ............................ None
338
+ moe_pad_expert_input_to_capacity ................ False
339
+ moe_per_layer_logging ........................... False
340
+ moe_permute_fusion .............................. False
341
+ moe_received_token_capacity ..................... None
342
+ moe_router_bias_update_rate ..................... 0.001
343
+ moe_router_dtype ................................ fp32
344
+ moe_router_enable_expert_bias ................... False
345
+ moe_router_force_load_balancing ................. False
346
+ moe_router_group_topk ........................... None
347
+ moe_router_load_balancing_type .................. aux_loss
348
+ moe_router_num_groups ........................... None
349
+ moe_router_padding_for_fp8 ...................... False
350
+ moe_router_pre_softmax .......................... False
351
+ moe_router_score_function ....................... softmax
352
+ moe_router_topk ................................. 8
353
+ moe_router_topk_scaling_factor .................. None
354
+ moe_shared_expert_intermediate_size ............. None
355
+ moe_shared_expert_overlap ....................... False
356
+ moe_token_dispatcher_type ....................... flex
357
+ moe_token_drop_policy ........................... probs
358
+ moe_upcycling_granularity ....................... 1
359
+ moe_use_device_initiated_grouped_gemm ........... True
360
+ moe_use_legacy_grouped_gemm ..................... False
361
+ moe_use_upcycling ............................... False
362
+ moe_z_loss_coeff ................................ None
363
+ mrope_section ................................... None
364
+ mscale .......................................... 1.0
365
+ mscale_all_dim .................................. 0.0
366
+ mtp_loss_scaling_factor ......................... 0.1
367
+ mtp_num_layers .................................. None
368
+ multi_latent_attention .......................... False
369
+ nccl_all_reduce_for_prefill ..................... False
370
+ nccl_communicator_config_path ................... None
371
+ nccl_ub ......................................... False
372
+ no_load_optim ................................... None
373
+ no_load_rng ..................................... None
374
+ no_persist_layer_norm ........................... False
375
+ no_rope_freq .................................... None
376
+ no_save_optim ................................... None
377
+ no_save_rng ..................................... None
378
+ non_persistent_ckpt_type ........................ None
379
+ non_persistent_global_ckpt_dir .................. None
380
+ non_persistent_local_ckpt_algo .................. fully_parallel
381
+ non_persistent_local_ckpt_dir ................... None
382
+ non_persistent_save_interval .................... None
383
+ norm_epsilon .................................... 1e-06
384
+ normalization ................................... RMSNorm
385
+ num_attention_heads ............................. 64
386
+ num_channels .................................... 3
387
+ num_classes ..................................... 1000
388
+ num_dataset_builder_threads ..................... 1
389
+ num_distributed_optimizer_instances ............. 1
390
+ num_experts ..................................... 128
391
+ num_layers ...................................... 12
392
+ num_layers_at_end_in_bf16 ....................... 1
393
+ num_layers_at_start_in_bf16 ..................... 1
394
+ num_layers_per_virtual_pipeline_stage ........... None
395
+ num_query_groups ................................ 4
396
+ num_virtual_stages_per_pipeline_rank ............ None
397
+ num_workers ..................................... 2
398
+ object_storage_cache_path ....................... None
399
+ one_logger_async ................................ False
400
+ one_logger_project .............................. megatron-lm
401
+ one_logger_run_name ............................. None
402
+ onnx_safe ....................................... None
403
+ openai_gelu ..................................... False
404
+ optimizer ....................................... adam
405
+ optimizer_cpu_offload ........................... False
406
+ optimizer_offload_fraction ...................... 1.0
407
+ output_bert_embeddings .......................... False
408
+ overlap_cpu_optimizer_d2h_h2d ................... False
409
+ overlap_grad_reduce ............................. True
410
+ overlap_moe_expert_parallel_comm ................ False
411
+ overlap_p2p_comm ................................ False
412
+ overlap_p2p_comm_warmup_flush ................... False
413
+ overlap_param_gather ............................ True
414
+ overlap_param_gather_with_optimizer_step ........ False
415
+ override_opt_param_scheduler .................... False
416
+ params_dtype .................................... torch.bfloat16
417
+ patch_dim ....................................... 16
418
+ per_split_data_args_path ........................ None
419
+ perform_initialization .......................... True
420
+ pin_cpu_grads ................................... True
421
+ pin_cpu_params .................................. True
422
+ pipeline_model_parallel_comm_backend ............ None
423
+ pipeline_model_parallel_layout .................. None
424
+ pipeline_model_parallel_size .................... 1
425
+ position_embedding_type ......................... rope
426
+ pretrained_checkpoint ........................... None
427
+ profile ......................................... False
428
+ profile_ranks ................................... [0]
429
+ profile_step_end ................................ 12
430
+ profile_step_start .............................. 10
431
+ q_lora_rank ..................................... None
432
+ qk_head_dim ..................................... 128
433
+ qk_l2_norm ...................................... False
434
+ qk_layernorm .................................... True
435
+ qk_pos_emb_head_dim ............................. 64
436
+ query_in_block_prob ............................. 0.1
437
+ rampup_batch_size ............................... None
438
+ rank ............................................ 0
439
+ recompute_granularity ........................... None
440
+ recompute_method ................................ None
441
+ recompute_modules ............................... None
442
+ recompute_num_layers ............................ None
443
+ record_cuda_graph_memory_history ................ False
444
+ record_full_program_memory_history .............. False
445
+ record_iteration_memory_history ................. True
446
+ record_memory_history ........................... False
447
+ relative_attention_max_distance ................. 128
448
+ relative_attention_num_buckets .................. 32
449
+ replication ..................................... False
450
+ replication_factor .............................. 2
451
+ replication_jump ................................ None
452
+ rerun_mode ...................................... validate_results
453
+ reset_attention_mask ............................ False
454
+ reset_position_ids .............................. False
455
+ result_rejected_tracker_filename ................ None
456
+ retriever_report_topk_accuracies ................ []
457
+ retriever_score_scaling ......................... False
458
+ retriever_seq_length ............................ 256
459
+ retro_add_retriever ............................. False
460
+ retro_attention_gate ............................ 1
461
+ retro_cyclic_train_iters ........................ None
462
+ retro_encoder_attention_dropout ................. 0.1
463
+ retro_encoder_hidden_dropout .................... 0.1
464
+ retro_encoder_layers ............................ 2
465
+ retro_num_neighbors ............................. 2
466
+ retro_num_retrieved_chunks ...................... 2
467
+ retro_project_dir ............................... None
468
+ retro_verify_neighbor_count ..................... True
469
+ reuse_grad_buf_for_mxfp8_param_ag ............... False
470
+ rope_scaling_factor ............................. 8.0
471
+ rope_type ....................................... None
472
+ rotary_base ..................................... 1000000
473
+ rotary_interleaved .............................. False
474
+ rotary_percent .................................. 1.0
475
+ rotary_scaling_factor ........................... 1.0
476
+ rotary_seq_len_interpolation_factor ............. None
477
+ run_workload_inspector_server ................... False
478
+ sample_rate ..................................... 1.0
479
+ save ............................................ None
480
+ save_interval ................................... None
481
+ save_retain_interval ............................ None
482
+ scatter_gather_tensors_in_pipeline .............. True
483
+ seed ............................................ 1234
484
+ seq_length ...................................... 8192
485
+ sequence_parallel ............................... False
486
+ sft ............................................. False
487
+ sft_tokenizer_prompt_format ..................... nemotron-h-aligned
488
+ sgd_momentum .................................... 0.9
489
+ short_seq_prob .................................. 0.1
490
+ skip_train ...................................... False
491
+ skipped_train_samples ........................... 0
492
+ spec ............................................ None
493
+ split ........................................... 100,0,0
494
+ squared_relu .................................... False
495
+ start_weight_decay .............................. 0.1
496
+ straggler_ctrlr_port ............................ 65535
497
+ straggler_minmax_count .......................... 1
498
+ suggested_communication_unit_size ............... None
499
+ swiglu .......................................... True
500
+ swin_backbone_type .............................. tiny
501
+ symmetric_ar_type ............................... None
502
+ te_rng_tracker .................................. False
503
+ tensor_model_parallel_size ...................... 1
504
+ tensorboard_dir ................................. None
505
+ tensorboard_log_interval ........................ 1
506
+ tensorboard_queue_size .......................... 1000
507
+ test_data_path .................................. None
508
+ test_mode ....................................... False
509
+ tiktoken_num_special_tokens ..................... 1000
510
+ tiktoken_pattern ................................ None
511
+ tiktoken_special_tokens ......................... None
512
+ timing_log_level ................................ 0
513
+ timing_log_option ............................... minmax
514
+ titles_data_path ................................ None
515
+ tokenizer_model ................................. None
516
+ tokenizer_type .................................. GPT2BPETokenizer
517
+ torch_fsdp2_reshard_after_forward ............... True
518
+ tp_comm_bootstrap_backend ....................... nccl
519
+ tp_comm_bulk_dgrad .............................. True
520
+ tp_comm_bulk_wgrad .............................. True
521
+ tp_comm_overlap ................................. False
522
+ tp_comm_overlap_ag .............................. True
523
+ tp_comm_overlap_cfg ............................. None
524
+ tp_comm_overlap_rs .............................. True
525
+ tp_comm_overlap_rs_dgrad ........................ False
526
+ tp_comm_split_ag ................................ True
527
+ tp_comm_split_rs ................................ True
528
+ train_data_path ................................. None
529
+ train_iters ..................................... 5
530
+ train_samples ................................... None
531
+ train_sync_interval ............................. None
532
+ transformer_impl ................................ transformer_engine
533
+ transformer_pipeline_model_parallel_size ........ 1
534
+ untie_embeddings_and_output_weights ............. True
535
+ use_checkpoint_args ............................. False
536
+ use_checkpoint_opt_param_scheduler .............. False
537
+ use_cpu_initialization .......................... None
538
+ use_custom_fsdp ................................. False
539
+ use_dist_ckpt ................................... True
540
+ use_dist_ckpt_deprecated ........................ False
541
+ use_distributed_optimizer ....................... True
542
+ use_flash_attn .................................. False
543
+ use_legacy_models ............................... False
544
+ use_mp_args_from_checkpoint_args ................ False
545
+ use_one_sent_docs ............................... False
546
+ use_persistent_ckpt_worker ...................... False
547
+ use_precision_aware_optimizer ................... False
548
+ use_pytorch_profiler ............................ False
549
+ use_ring_exchange_p2p ........................... False
550
+ use_rope_scaling ................................ False
551
+ use_rotary_position_embeddings .................. False
552
+ use_sharp ....................................... False
553
+ use_tokenizer_model_from_checkpoint_args ........ True
554
+ use_torch_fsdp2 ................................. False
555
+ use_torch_optimizer_for_cpu_offload ............. False
556
+ use_tp_pp_dp_mapping ............................ False
557
+ v_head_dim ...................................... 128
558
+ valid_data_path ................................. None
559
+ variable_seq_lengths ............................ False
560
+ virtual_pipeline_model_parallel_size ............ None
561
+ vision_backbone_type ............................ vit
562
+ vision_pretraining .............................. False
563
+ vision_pretraining_type ......................... classify
564
+ vocab_extra_ids ................................. 0
565
+ vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json
566
+ vocab_size ...................................... None
567
+ wandb_exp_name ..................................
568
+ wandb_project ...................................
569
+ wandb_save_dir ..................................
570
+ weight_decay .................................... 0.1
571
+ weight_decay_incr_style ......................... constant
572
+ wgrad_deferral_limit ............................ 0
573
+ world_size ...................................... 8
574
+ yaml_cfg ........................................ None
575
+ -------------------- end of arguments ---------------------
576
+ INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1
577
+ > building GPT2BPETokenizer tokenizer ...
578
+ > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304)
579
+ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS
580
+ > initializing torch distributed ...
581
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
582
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
583
+ [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
584
+ [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
585
+ [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
586
+ [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
587
+ [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
588
+ [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
589
+ [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
590
+ [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
591
+ [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
592
+ [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
593
+ [Gloo] Rank [Gloo] Rank 4 is connected to 17 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is :
594
+ 7
595
+ [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
596
+ [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
597
+ [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
598
+ [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
599
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
600
+ > initialized tensor model parallel with size 1
601
+ > initialized pipeline model parallel with size 1
602
+ > setting random seeds to 1234 ...
603
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
604
+ [Gloo] Rank [Gloo] Rank 0[Gloo] Rank 0 is connected to 0 is connected to 0 is connected to 0 peer ranks. 0 peer ranks. Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : 0Expected number of connected peer ranks is : 0
605
+ 0
606
+
607
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
608
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
609
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
610
+ [rank6]:[W830 00:01:56.441895105 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
611
+ [rank1]:[W830 00:01:56.442181308 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
612
+ [rank4]:[W830 00:01:56.442202971 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
613
+ [rank3]:[W830 00:01:56.443016796 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
614
+ [rank7]:[W830 00:01:56.447548869 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
615
+ [rank2]:[W830 00:01:56.447655991 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
616
+ [rank5]:[W830 00:01:56.448071498 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
617
+ > compiling dataset index builder ...
618
+ make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets'
619
+ make: Nothing to be done for 'default'.
620
+ make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets'
621
+ >>> done with dataset index builder. Compilation time: 0.034 seconds
622
+ WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations.
623
+ > compiling and loading fused kernels ...
624
+ [rank0]:[W830 00:01:56.608074872 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
625
+ >>> done with compiling and loading fused kernels. Compilation time: 3.780 seconds
626
+ time to initialize megatron (seconds): 7.286
627
+ [after megatron is initialized] datetime: 2026-08-30 00:02:02
628
+ building GPT model ...
629
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
630
+ warnings.warn(
631
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
632
+ warnings.warn(
633
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
634
+ warnings.warn(
635
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
636
+ warnings.warn(
637
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
638
+ warnings.warn(
639
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
640
+ warnings.warn(
641
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
642
+ warnings.warn(
643
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
644
+ warnings.warn(
645
+ > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4898003968
646
+ INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False)
647
+ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 14
648
+ Params for bucket 1 (206045184 elements, 206045184 padded size):
649
+ module.output_layer.weight
650
+ Params for bucket 2 (71840000 elements, 71840000 padded size):
651
+ module.decoder.final_layernorm.weight
652
+ module.decoder.layers.11.pre_mlp_layernorm.weight
653
+ module.decoder.layers.11.self_attention.q_layernorm.weight
654
+ module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight
655
+ module.decoder.layers.11.mlp.router.weight
656
+ module.decoder.layers.11.self_attention.k_layernorm.weight
657
+ module.decoder.layers.11.self_attention.linear_qkv.weight
658
+ module.decoder.layers.11.self_attention.linear_proj.weight
659
+ Params for bucket 3 (71835904 elements, 71835904 padded size):
660
+ module.decoder.layers.10.self_attention.linear_proj.weight
661
+ module.decoder.layers.10.mlp.router.weight
662
+ module.decoder.layers.10.self_attention.k_layernorm.weight
663
+ module.decoder.layers.10.self_attention.q_layernorm.weight
664
+ module.decoder.layers.10.pre_mlp_layernorm.weight
665
+ module.decoder.layers.10.self_attention.linear_qkv.weight
666
+ module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight
667
+ Params for bucket 4 (71835904 elements, 71835904 padded size):
668
+ module.decoder.layers.9.mlp.router.weight
669
+ module.decoder.layers.9.self_attention.q_layernorm.weight
670
+ module.decoder.layers.9.self_attention.k_layernorm.weight
671
+ module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight
672
+ module.decoder.layers.9.self_attention.linear_proj.weight
673
+ module.decoder.layers.9.self_attention.linear_qkv.weight
674
+ module.decoder.layers.9.pre_mlp_layernorm.weight
675
+ Params for bucket 5 (71835904 elements, 71835904 padded size):
676
+ module.decoder.layers.8.mlp.router.weight
677
+ module.decoder.layers.8.self_attention.linear_qkv.weight
678
+ module.decoder.layers.8.self_attention.k_layernorm.weight
679
+ module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight
680
+ module.decoder.layers.8.pre_mlp_layernorm.weight
681
+ module.decoder.layers.8.self_attention.q_layernorm.weight
682
+ module.decoder.layers.8.self_attention.linear_proj.weight
683
+ Params for bucket 6 (71835904 elements, 71835904 padded size):
684
+ module.decoder.layers.7.mlp.router.weight
685
+ module.decoder.layers.7.pre_mlp_layernorm.weight
686
+ module.decoder.layers.7.self_attention.q_layernorm.weight
687
+ module.decoder.layers.7.self_attention.k_layernorm.weight
688
+ module.decoder.layers.7.self_attention.linear_qkv.weight
689
+ module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight
690
+ module.decoder.layers.7.self_attention.linear_proj.weight
691
+ Params for bucket 7 (71835904 elements, 71835904 padded size):
692
+ module.decoder.layers.6.mlp.router.weight
693
+ module.decoder.layers.6.self_attention.linear_qkv.weight
694
+ module.decoder.layers.6.self_attention.k_layernorm.weight
695
+ module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight
696
+ module.decoder.layers.6.pre_mlp_layernorm.weight
697
+ module.decoder.layers.6.self_attention.q_layernorm.weight
698
+ module.decoder.layers.6.self_attention.linear_proj.weight
699
+ Params for bucket 8 (71835904 elements, 71835904 padded size):
700
+ module.decoder.layers.5.mlp.router.weight
701
+ module.decoder.layers.5.self_attention.linear_qkv.weight
702
+ module.decoder.layers.5.self_attention.k_layernorm.weight
703
+ module.decoder.layers.5.self_attention.q_layernorm.weight
704
+ module.decoder.layers.5.pre_mlp_layernorm.weight
705
+ module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight
706
+ module.decoder.layers.5.self_attention.linear_proj.weight
707
+ Params for bucket 9 (71835904 elements, 71835904 padded size):
708
+ module.decoder.layers.4.mlp.router.weight
709
+ module.decoder.layers.4.pre_mlp_layernorm.weight
710
+ module.decoder.layers.4.self_attention.linear_qkv.weight
711
+ module.decoder.layers.4.self_attention.q_layernorm.weight
712
+ module.decoder.layers.4.self_attention.k_layernorm.weight
713
+ module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight
714
+ module.decoder.layers.4.self_attention.linear_proj.weight
715
+ Params for bucket 10 (71835904 elements, 71835904 padded size):
716
+ module.decoder.layers.3.mlp.router.weight
717
+ module.decoder.layers.3.pre_mlp_layernorm.weight
718
+ module.decoder.layers.3.self_attention.k_layernorm.weight
719
+ module.decoder.layers.3.self_attention.q_layernorm.weight
720
+ module.decoder.layers.3.self_attention.linear_qkv.weight
721
+ module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight
722
+ module.decoder.layers.3.self_attention.linear_proj.weight
723
+ Params for bucket 11 (71835904 elements, 71835904 padded size):
724
+ module.decoder.layers.2.mlp.router.weight
725
+ module.decoder.layers.2.self_attention.linear_qkv.weight
726
+ module.decoder.layers.2.self_attention.k_layernorm.weight
727
+ module.decoder.layers.2.pre_mlp_layernorm.weight
728
+ module.decoder.layers.2.self_attention.q_layernorm.weight
729
+ module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight
730
+ module.decoder.layers.2.self_attention.linear_proj.weight
731
+ Params for bucket 12 (71835904 elements, 71835904 padded size):
732
+ module.decoder.layers.1.pre_mlp_layernorm.weight
733
+ module.decoder.layers.1.mlp.router.weight
734
+ module.decoder.layers.1.self_attention.linear_qkv.weight
735
+ module.decoder.layers.1.self_attention.q_layernorm.weight
736
+ module.decoder.layers.1.self_attention.k_layernorm.weight
737
+ module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight
738
+ module.decoder.layers.1.self_attention.linear_proj.weight
739
+ Params for bucket 13 (71835904 elements, 71835904 padded size):
740
+ module.decoder.layers.0.mlp.router.weight
741
+ module.decoder.layers.0.self_attention.linear_qkv.weight
742
+ module.decoder.layers.0.self_attention.linear_proj.weight
743
+ module.decoder.layers.0.self_attention.q_layernorm.weight
744
+ module.decoder.layers.0.self_attention.k_layernorm.weight
745
+ module.decoder.layers.0.pre_mlp_layernorm.weight
746
+ module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight
747
+ Params for bucket 14 (206045184 elements, 206045184 padded size):
748
+ module.embedding.word_embeddings.weight
749
+ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76
750
+ Params for bucket 1 (44040192 elements, 44040192 padded size):
751
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight1
752
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight2
753
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight4
754
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight6
755
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight0
756
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight3
757
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight5
758
+ Params for bucket 2 (44040192 elements, 44040192 padded size):
759
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight8
760
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight10
761
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight12
762
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight7
763
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight9
764
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight11
765
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight13
766
+ Params for bucket 3 (50331648 elements, 50331648 padded size):
767
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight14
768
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight0
769
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight15
770
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight2
771
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight1
772
+ Params for bucket 4 (50331648 elements, 50331648 padded size):
773
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight3
774
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight4
775
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight5
776
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight6
777
+ Params for bucket 5 (50331648 elements, 50331648 padded size):
778
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight7
779
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight10
780
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight8
781
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight9
782
+ Params for bucket 6 (50331648 elements, 50331648 padded size):
783
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight11
784
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight13
785
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight14
786
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight12
787
+ Params for bucket 7 (44040192 elements, 44040192 padded size):
788
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight2
789
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight1
790
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight4
791
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight15
792
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight3
793
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight0
794
+ Params for bucket 8 (44040192 elements, 44040192 padded size):
795
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight6
796
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight8
797
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight10
798
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight5
799
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight9
800
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight7
801
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight11
802
+ Params for bucket 9 (50331648 elements, 50331648 padded size):
803
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight12
804
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight14
805
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight1
806
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight0
807
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight15
808
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight13
809
+ Params for bucket 10 (50331648 elements, 50331648 padded size):
810
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight2
811
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight5
812
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight4
813
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight3
814
+ Params for bucket 11 (50331648 elements, 50331648 padded size):
815
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight6
816
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight7
817
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight9
818
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight8
819
+ Params for bucket 12 (50331648 elements, 50331648 padded size):
820
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight10
821
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight13
822
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight11
823
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight12
824
+ Params for bucket 13 (44040192 elements, 44040192 padded size):
825
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight0
826
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight15
827
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight14
828
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight2
829
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight1
830
+ Params for bucket 14 (44040192 elements, 44040192 padded size):
831
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight3
832
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight7
833
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight5
834
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight9
835
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight4
836
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight6
837
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight8
838
+ Params for bucket 15 (50331648 elements, 50331648 padded size):
839
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight13
840
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight11
841
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight15
842
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight10
843
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight14
844
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight12
845
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight0
846
+ Params for bucket 16 (50331648 elements, 50331648 padded size):
847
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight1
848
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight4
849
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight2
850
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight3
851
+ Params for bucket 17 (50331648 elements, 50331648 padded size):
852
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight5
853
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight6
854
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight7
855
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight8
856
+ Params for bucket 18 (50331648 elements, 50331648 padded size):
857
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight9
858
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight12
859
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight10
860
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight11
861
+ Params for bucket 19 (44040192 elements, 44040192 padded size):
862
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight13
863
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight0
864
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight15
865
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight14
866
+ Params for bucket 20 (44040192 elements, 44040192 padded size):
867
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight1
868
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight4
869
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight6
870
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight5
871
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight2
872
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight3
873
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight7
874
+ Params for bucket 21 (44040192 elements, 44040192 padded size):
875
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight8
876
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight10
877
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight12
878
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight14
879
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight11
880
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight9
881
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight13
882
+ Params for bucket 22 (44040192 elements, 44040192 padded size):
883
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight15
884
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight0
885
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight2
886
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight1
887
+ Params for bucket 23 (50331648 elements, 50331648 padded size):
888
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight3
889
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight4
890
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight5
891
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight6
892
+ Params for bucket 24 (50331648 elements, 50331648 padded size):
893
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight7
894
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight10
895
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight8
896
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight9
897
+ Params for bucket 25 (50331648 elements, 50331648 padded size):
898
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight11
899
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight13
900
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight14
901
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight12
902
+ Params for bucket 26 (44040192 elements, 44040192 padded size):
903
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight15
904
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight1
905
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight2
906
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight4
907
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight3
908
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight0
909
+ Params for bucket 27 (44040192 elements, 44040192 padded size):
910
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight6
911
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight8
912
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight10
913
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight5
914
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight9
915
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight7
916
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight11
917
+ Params for bucket 28 (50331648 elements, 50331648 padded size):
918
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight12
919
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight0
920
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight14
921
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight15
922
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight13
923
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight1
924
+ Params for bucket 29 (50331648 elements, 50331648 padded size):
925
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight2
926
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight4
927
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight5
928
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight3
929
+ Params for bucket 30 (50331648 elements, 50331648 padded size):
930
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight6
931
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight8
932
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight9
933
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight7
934
+ Params for bucket 31 (50331648 elements, 50331648 padded size):
935
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight10
936
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight11
937
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight12
938
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight13
939
+ Params for bucket 32 (44040192 elements, 44040192 padded size):
940
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight14
941
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight1
942
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight0
943
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight2
944
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight15
945
+ Params for bucket 33 (44040192 elements, 44040192 padded size):
946
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight4
947
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight6
948
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight8
949
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight3
950
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight7
951
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight5
952
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight9
953
+ Params for bucket 34 (50331648 elements, 50331648 padded size):
954
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight10
955
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight12
956
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight14
957
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight11
958
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight13
959
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight15
960
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight0
961
+ Params for bucket 35 (50331648 elements, 50331648 padded size):
962
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight1
963
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight4
964
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight2
965
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight3
966
+ Params for bucket 36 (50331648 elements, 50331648 padded size):
967
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight5
968
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight7
969
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight8
970
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight6
971
+ Params for bucket 37 (50331648 elements, 50331648 padded size):
972
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight9
973
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight10
974
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight11
975
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight12
976
+ Params for bucket 38 (44040192 elements, 44040192 padded size):
977
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight13
978
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight14
979
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight15
980
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight0
981
+ Params for bucket 39 (44040192 elements, 44040192 padded size):
982
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight3
983
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight5
984
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight7
985
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight1
986
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight2
987
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight4
988
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight6
989
+ Params for bucket 40 (44040192 elements, 44040192 padded size):
990
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight9
991
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight11
992
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight13
993
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight8
994
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight12
995
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight10
996
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight14
997
+ Params for bucket 41 (44040192 elements, 44040192 padded size):
998
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight15
999
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight1
1000
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight0
1001
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight2
1002
+ Params for bucket 42 (50331648 elements, 50331648 padded size):
1003
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight3
1004
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight6
1005
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight4
1006
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight5
1007
+ Params for bucket 43 (50331648 elements, 50331648 padded size):
1008
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight7
1009
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight10
1010
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight9
1011
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight8
1012
+ Params for bucket 44 (50331648 elements, 50331648 padded size):
1013
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight11
1014
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight13
1015
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight12
1016
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight14
1017
+ Params for bucket 45 (44040192 elements, 44040192 padded size):
1018
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight1
1019
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight4
1020
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight15
1021
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight3
1022
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight2
1023
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight0
1024
+ Params for bucket 46 (44040192 elements, 44040192 padded size):
1025
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight8
1026
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight6
1027
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight10
1028
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight5
1029
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight9
1030
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight7
1031
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight11
1032
+ Params for bucket 47 (50331648 elements, 50331648 padded size):
1033
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight12
1034
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight14
1035
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight1
1036
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight15
1037
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight13
1038
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight0
1039
+ Params for bucket 48 (50331648 elements, 50331648 padded size):
1040
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight2
1041
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight4
1042
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight3
1043
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight5
1044
+ Params for bucket 49 (50331648 elements, 50331648 padded size):
1045
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight6
1046
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight7
1047
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight8
1048
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight9
1049
+ Params for bucket 50 (50331648 elements, 50331648 padded size):
1050
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight10
1051
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight13
1052
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight11
1053
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight12
1054
+ Params for bucket 51 (44040192 elements, 44040192 padded size):
1055
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight2
1056
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight0
1057
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight15
1058
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight14
1059
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight1
1060
+ Params for bucket 52 (44040192 elements, 44040192 padded size):
1061
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight3
1062
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight7
1063
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight5
1064
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight9
1065
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight6
1066
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight4
1067
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight8
1068
+ Params for bucket 53 (50331648 elements, 50331648 padded size):
1069
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight13
1070
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight11
1071
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight15
1072
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight0
1073
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight10
1074
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight12
1075
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight14
1076
+ Params for bucket 54 (50331648 elements, 50331648 padded size):
1077
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight1
1078
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight3
1079
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight4
1080
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight2
1081
+ Params for bucket 55 (50331648 elements, 50331648 padded size):
1082
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight5
1083
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight8
1084
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight6
1085
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight7
1086
+ Params for bucket 56 (50331648 elements, 50331648 padded size):
1087
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight9
1088
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight12
1089
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight10
1090
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight11
1091
+ Params for bucket 57 (44040192 elements, 44040192 padded size):
1092
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight13
1093
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight15
1094
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight0
1095
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight14
1096
+ Params for bucket 58 (44040192 elements, 44040192 padded size):
1097
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight5
1098
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight2
1099
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight3
1100
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight7
1101
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight1
1102
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight4
1103
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight6
1104
+ Params for bucket 59 (44040192 elements, 44040192 padded size):
1105
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight11
1106
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight9
1107
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight13
1108
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight8
1109
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight12
1110
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight10
1111
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight14
1112
+ Params for bucket 60 (44040192 elements, 44040192 padded size):
1113
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight15
1114
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight1
1115
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight0
1116
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight2
1117
+ Params for bucket 61 (50331648 elements, 50331648 padded size):
1118
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight3
1119
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight5
1120
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight6
1121
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight4
1122
+ Params for bucket 62 (50331648 elements, 50331648 padded size):
1123
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight7
1124
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight8
1125
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight9
1126
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight10
1127
+ Params for bucket 63 (50331648 elements, 50331648 padded size):
1128
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight11
1129
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight12
1130
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight13
1131
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight14
1132
+ Params for bucket 64 (44040192 elements, 44040192 padded size):
1133
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight1
1134
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight2
1135
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight4
1136
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight15
1137
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight3
1138
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight0
1139
+ Params for bucket 65 (44040192 elements, 44040192 padded size):
1140
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight8
1141
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight6
1142
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight10
1143
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight5
1144
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight9
1145
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight7
1146
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight11
1147
+ Params for bucket 66 (50331648 elements, 50331648 padded size):
1148
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight12
1149
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight14
1150
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight0
1151
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight15
1152
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight13
1153
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight1
1154
+ Params for bucket 67 (50331648 elements, 50331648 padded size):
1155
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight2
1156
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight4
1157
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight5
1158
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight3
1159
+ Params for bucket 68 (50331648 elements, 50331648 padded size):
1160
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight6
1161
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight7
1162
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight8
1163
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight9
1164
+ Params for bucket 69 (50331648 elements, 50331648 padded size):
1165
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight10
1166
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight13
1167
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight11
1168
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight12
1169
+ Params for bucket 70 (44040192 elements, 44040192 padded size):
1170
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight14
1171
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight0
1172
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight1
1173
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight2
1174
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight15
1175
+ Params for bucket 71 (44040192 elements, 44040192 padded size):
1176
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight3
1177
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight5
1178
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight7
1179
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight9
1180
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight6
1181
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight4
1182
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight8
1183
+ Params for bucket 72 (50331648 elements, 50331648 padded size):
1184
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight13
1185
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight11
1186
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight15
1187
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight0
1188
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight10
1189
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight12
1190
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight14
1191
+ Params for bucket 73 (50331648 elements, 50331648 padded size):
1192
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight1
1193
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight4
1194
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight3
1195
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight2
1196
+ Params for bucket 74 (50331648 elements, 50331648 padded size):
1197
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight5
1198
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight6
1199
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight7
1200
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight8
1201
+ Params for bucket 75 (50331648 elements, 50331648 padded size):
1202
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight9
1203
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight10
1204
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight11
1205
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight12
1206
+ Params for bucket 76 (37748736 elements, 37748736 padded size):
1207
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight13
1208
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight14
1209
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight15
1210
+ INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=<megatron.core.timers.Timers object at 0x7c083d791670>, config_logger_dir='')
1211
+ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine
1212
+ [after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 00:02:05
1213
+ > building train, validation, and test datasets ...
1214
+ > datasets target sizes (minimum size):
1215
+ train: 40
1216
+ validation: 0
1217
+ test: 0
1218
+ INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None]
1219
+ > building train, validation, and test datasets for GPT ...
1220
+ INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(40, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=<megatron.training.tokenizer.tokenizer._GPT2BPETokenizer object at 0x7c05970d8980>, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None)
1221
+ INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx
1222
+ INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths
1223
+ INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers
1224
+ INFO:megatron.core.datasets.indexed_dataset: Extract the document indices
1225
+ INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029
1226
+ INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350
1227
+ INFO:megatron.core.datasets.gpt_dataset:Build and save the GPTDataset train indices
1228
+ INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536
1229
+ INFO:megatron.core.datasets.gpt_dataset:> total number of epochs: 1
1230
+ > finished creating GPT datasets ...
1231
+ [after dataloaders are built] datetime: 2026-08-30 00:02:05
1232
+ done with setup ...
1233
+ (min, max) time across ranks (ms):
1234
+ model-and-optimizer-setup ......................: (3397.41, 3407.14)
1235
+ train/valid/test-data-iterators-setup ..........: (123.48, 126.00)
1236
+ training ...
1237
+ Overwriting rerun_state_machine.current_iteration from -1 to 0...
1238
+ [before the start of training step] datetime: 2026-08-30 00:02:05
1239
+ INFO:moonep.api:MoonEP comm buffer sizes:
1240
+ config : R=8, S=8192, H=4096, E=128, K=8, NvS=69600, NvS_padded=69632
1241
+ sm_count : num_sms=32, num_sms_dedup=148
1242
+ hidden_buffer_size : 544.00 MiB
1243
+ meta_buffer_size : 2.00 MiB
1244
+ local_temp_buffer_size : 2.52 MiB
1245
+ total_size : 548.52 MiB
1246
+ [MoonEP selfcheck 1/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [5376, 640, 256, 2816, 1280, 5632, 256, 256, 768, 1920, 14208, 4992, 14848, 896, 1664, 3584, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[127, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [43, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [127, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [64, 74, 68, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [107, 99, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [29, 17, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]]
1247
+ [MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [9344, 2688, 128, 24192, 128, 0, 128, 256, 128, 768, 10496, 128, 128, 2432, 128, 1536, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[69, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [88, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [55, 52, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [63, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [124, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [58, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]]
1248
+ [MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [128, 128, 128, 0, 128, 44416, 128, 128, 128, 6912, 128, 256, 128, 128, 128, 128, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[78, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [111, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [62, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [57, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [106, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [121, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [57, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]]
1249
+ [MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [2304, 128, 128, 512, 128, 12416, 128, 1152, 1280, 16768, 640, 29056, 128, 128, 128, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [118, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [118, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [39, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [73, 79, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [86, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]]
1250
+ WARNING:megatron.core.rerun_state_machine:Result validation enabled
1251
+ [2026-08-30 00:02:27] iteration 1/ 5 | consumed samples: 8 | elapsed time per iteration (ms): 21524.9 | throughput per GPU (TFLOP/s/GPU): 8.4 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115993E+01 | loss scale: 1.0 | grad norm: 44.807 | number of skipped iterations: 0 | number of nan iterations: 0 |
1252
+ Number of parameters in transformer block in billions: 29.85
1253
+ Number of parameters in embedding layers in billions: 0.41
1254
+ Total number of parameters in billions: 30.26
1255
+ Number of parameters in most loaded shard in billions: 30.2590
1256
+ Theoretical memory footprints: weight and optimizer=216428.96 MB
1257
+ [Rank 0] (after 1 iterations) memory (MB) | allocated: 67175.32958984375 | max allocated: 67175.330078125 | reserved: 68476.0 | max reserved: 68476.0
1258
+ [2026-08-30 00:02:28] iteration 2/ 5 | consumed samples: 16 | elapsed time per iteration (ms): 776.8 | throughput per GPU (TFLOP/s/GPU): 232.8 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117600E+01 | loss scale: 1.0 | grad norm: 44.336 | number of skipped iterations: 0 | number of nan iterations: 0 |
1259
+ [2026-08-30 00:02:28] iteration 3/ 5 | consumed samples: 24 | elapsed time per iteration (ms): 398.9 | throughput per GPU (TFLOP/s/GPU): 453.3 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.686371E+00 | loss scale: 1.0 | grad norm: 62.939 | number of skipped iterations: 0 | number of nan iterations: 0 |
1260
+ [2026-08-30 00:02:29] iteration 4/ 5 | consumed samples: 32 | elapsed time per iteration (ms): 394.8 | throughput per GPU (TFLOP/s/GPU): 458.0 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338075E+01 | loss scale: 1.0 | grad norm: 20.301 | number of skipped iterations: 0 | number of nan iterations: 0 |
1261
+ [2026-08-30 00:02:29] iteration 5/ 5 | consumed samples: 40 | elapsed time per iteration (ms): 396.0 | throughput per GPU (TFLOP/s/GPU): 456.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.049910E+01 | loss scale: 1.0 | grad norm: 95.166 | number of skipped iterations: 0 | number of nan iterations: 0 |
1262
+ [after training is done] datetime: 2026-08-30 00:02:29
1263
+ [rank0]:[W830 00:02:30.006178917 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1264
+ [rank3]:[W830 00:02:31.929533701 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1265
+ [rank5]:[W830 00:02:31.099652422 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1266
+ [rank7]:[W830 00:02:31.115394431 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1267
+ [rank2]:[W830 00:02:31.141780136 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1268
+ [rank6]:[W830 00:02:32.425427924 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1269
+ [rank1]:[W830 00:02:32.626966226 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1270
+ [rank4]:[W830 00:02:32.092291203 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1271
+ W0830 00:02:38.142000 271073 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers
1272
+ W0830 00:02:38.143000 271073 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 271263 closing signal SIGTERM
1273
+ W0830 00:02:38.143000 271073 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 271266 closing signal SIGTERM
1274
+ W0830 00:02:38.144000 271073 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 271269 closing signal SIGTERM
1275
+ Traceback (most recent call last):
1276
+ File "/usr/local/bin/torchrun", line 8, in <module>
1277
+ sys.exit(main())
1278
+ ^^^^^^
1279
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
1280
+ return f(*args, **kwargs)
1281
+ ^^^^^^^^^^^^^^^^^^
1282
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main
1283
+ run(args)
1284
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run
1285
+ elastic_launch(
1286
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__
1287
+ return launch_agent(self._config, self._entrypoint, list(args))
1288
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1289
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent
1290
+ result = agent.run()
1291
+ ^^^^^^^^^^^
1292
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
1293
+ result = f(*args, **kwargs)
1294
+ ^^^^^^^^^^^^^^^^^^
1295
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run
1296
+ result = self._invoke_run(role)
1297
+ ^^^^^^^^^^^^^^^^^^^^^^
1298
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run
1299
+ time.sleep(monitor_interval)
1300
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler
1301
+ raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
1302
+ torch.distributed.elastic.multiprocessing.api.SignalException: Process 271073 got signal: 15
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/mem_samples.csv ADDED
@@ -0,0 +1,232 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0
2
+ 1, 0
3
+ 2, 0
4
+ 3, 0
5
+ 4, 0
6
+ 5, 0
7
+ 6, 0
8
+ 7, 0
9
+ 0, 4
10
+ 1, 4
11
+ 2, 1
12
+ 3, 1
13
+ 4, 1
14
+ 5, 1
15
+ 6, 1
16
+ 7, 1
17
+ 0, 4
18
+ 1, 4
19
+ 2, 4
20
+ 3, 4
21
+ 4, 4
22
+ 5, 4
23
+ 6, 4
24
+ 7, 1
25
+ 0, 1
26
+ 1, 1
27
+ 2, 1
28
+ 3, 1
29
+ 4, 1
30
+ 5, 4
31
+ 6, 4
32
+ 7, 4
33
+ 0, 0
34
+ 1, 0
35
+ 2, 0
36
+ 3, 0
37
+ 4, 0
38
+ 5, 0
39
+ 6, 0
40
+ 7, 0
41
+ 0, 4
42
+ 1, 4
43
+ 2, 1
44
+ 3, 1
45
+ 4, 1
46
+ 5, 1
47
+ 6, 1
48
+ 7, 1
49
+ 0, 4
50
+ 1, 4
51
+ 2, 4
52
+ 3, 4
53
+ 4, 4
54
+ 5, 1
55
+ 6, 1
56
+ 7, 1
57
+ 0, 4
58
+ 1, 4
59
+ 2, 4
60
+ 3, 4
61
+ 4, 4
62
+ 5, 4
63
+ 6, 4
64
+ 7, 4
65
+ 0, 4
66
+ 1, 4
67
+ 2, 4
68
+ 3, 4
69
+ 4, 4
70
+ 5, 4
71
+ 6, 4
72
+ 7, 4
73
+ 0, 821
74
+ 1, 819
75
+ 2, 819
76
+ 3, 819
77
+ 4, 821
78
+ 5, 819
79
+ 6, 819
80
+ 7, 819
81
+ 0, 1537
82
+ 1, 1537
83
+ 2, 1537
84
+ 3, 1537
85
+ 4, 1537
86
+ 5, 1537
87
+ 6, 1537
88
+ 7, 1537
89
+ 0, 1975
90
+ 1, 1975
91
+ 2, 1975
92
+ 3, 1975
93
+ 4, 1975
94
+ 5, 1975
95
+ 6, 1975
96
+ 7, 1975
97
+ 0, 46579
98
+ 1, 46573
99
+ 2, 46575
100
+ 3, 46573
101
+ 4, 46583
102
+ 5, 46579
103
+ 6, 46591
104
+ 7, 46579
105
+ 0, 47547
106
+ 1, 47549
107
+ 2, 47549
108
+ 3, 47549
109
+ 4, 47549
110
+ 5, 47549
111
+ 6, 47549
112
+ 7, 47549
113
+ 0, 48005
114
+ 1, 47851
115
+ 2, 48079
116
+ 3, 47845
117
+ 4, 47835
118
+ 5, 48065
119
+ 6, 47855
120
+ 7, 48073
121
+ 0, 48699
122
+ 1, 48349
123
+ 2, 48721
124
+ 3, 48355
125
+ 4, 48505
126
+ 5, 48715
127
+ 6, 48369
128
+ 7, 48743
129
+ 0, 49251
130
+ 1, 48907
131
+ 2, 49279
132
+ 3, 48905
133
+ 4, 49041
134
+ 5, 49279
135
+ 6, 48905
136
+ 7, 49279
137
+ 0, 49251
138
+ 1, 48907
139
+ 2, 49279
140
+ 3, 48907
141
+ 4, 49043
142
+ 5, 49279
143
+ 6, 48907
144
+ 7, 49279
145
+ 0, 58515
146
+ 1, 58601
147
+ 2, 58703
148
+ 3, 58539
149
+ 4, 58779
150
+ 5, 58703
151
+ 6, 58539
152
+ 7, 58703
153
+ 0, 83559
154
+ 1, 83563
155
+ 2, 83551
156
+ 3, 83543
157
+ 4, 83571
158
+ 5, 83553
159
+ 6, 83543
160
+ 7, 83553
161
+ 0, 83947
162
+ 1, 83951
163
+ 2, 83941
164
+ 3, 83931
165
+ 4, 83959
166
+ 5, 83941
167
+ 6, 83931
168
+ 7, 83941
169
+ 0, 84339
170
+ 1, 84343
171
+ 2, 84341
172
+ 3, 84323
173
+ 4, 84351
174
+ 5, 84341
175
+ 6, 84323
176
+ 7, 84341
177
+ 0, 88029
178
+ 1, 87985
179
+ 2, 87671
180
+ 3, 87773
181
+ 4, 87817
182
+ 5, 87931
183
+ 6, 87541
184
+ 7, 87831
185
+ 0, 95543
186
+ 1, 95455
187
+ 2, 95533
188
+ 3, 95309
189
+ 4, 95465
190
+ 5, 95551
191
+ 6, 95423
192
+ 7, 95579
193
+ 0, 116439
194
+ 1, 116343
195
+ 2, 116411
196
+ 3, 116193
197
+ 4, 116315
198
+ 5, 116411
199
+ 6, 116289
200
+ 7, 116411
201
+ 0, 116703
202
+ 1, 116611
203
+ 2, 116701
204
+ 3, 116459
205
+ 4, 116583
206
+ 5, 116701
207
+ 6, 116555
208
+ 7, 116701
209
+ 0, 116121
210
+ 1, 116611
211
+ 2, 116375
212
+ 3, 115877
213
+ 4, 116583
214
+ 5, 116439
215
+ 6, 116555
216
+ 7, 116375
217
+ 0, 114831
218
+ 1, 8500
219
+ 2, 114829
220
+ 3, 114587
221
+ 4, 115143
222
+ 5, 114829
223
+ 6, 115115
224
+ 7, 114829
225
+ 0, 1
226
+ 1, 3
227
+ 2, 196
228
+ 3, 114587
229
+ 4, 964
230
+ 5, 4
231
+ 6, 4
232
+ 7, 4
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260830T000343559627Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 69838439936,
3
+ "active_bytes_current_delta": 48030109696,
4
+ "active_bytes_peak": 69838443008,
5
+ "active_bytes_peak_delta": 46343805440,
6
+ "after": {
7
+ "active_bytes_current": 69838439936,
8
+ "active_bytes_peak": 69838443008,
9
+ "allocated_bytes_current": 69838439936,
10
+ "allocated_bytes_peak": 69838443008,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 1743649280,
13
+ "inactive_split_bytes_peak": 13251217920,
14
+ "iteration": 0,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 71605157888,
21
+ "reserved_bytes_peak": 71605157888,
22
+ "segment_count_current": 2597,
23
+ "segment_count_peak": 2597,
24
+ "timestamp_utc": "2026-08-30T00:03:43.559140Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 69838439936,
28
+ "allocated_bytes_current_delta": 48030109696,
29
+ "allocated_bytes_peak": 69838443008,
30
+ "allocated_bytes_peak_delta": 46343805440,
31
+ "before": {
32
+ "active_bytes_current": 21808330240,
33
+ "active_bytes_peak": 23494637568,
34
+ "allocated_bytes_current": 21808330240,
35
+ "allocated_bytes_peak": 23494637568,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 1876904448,
38
+ "inactive_split_bytes_peak": 1876904448,
39
+ "iteration": 0,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 25293750272,
46
+ "reserved_bytes_peak": 25293750272,
47
+ "segment_count_current": 888,
48
+ "segment_count_peak": 888,
49
+ "timestamp_utc": "2026-08-30T00:03:20.789954Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1743649280,
55
+ "inactive_split_bytes_current_delta": -133255168,
56
+ "inactive_split_bytes_peak": 13251217920,
57
+ "inactive_split_bytes_peak_delta": 11374313472,
58
+ "iteration": 0,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 71605157888,
68
+ "reserved_bytes_current_delta": 46311407616,
69
+ "reserved_bytes_peak": 71605157888,
70
+ "reserved_bytes_peak_delta": 46311407616,
71
+ "segment_count_current": 2597,
72
+ "segment_count_current_delta": 1709,
73
+ "segment_count_peak": 2597,
74
+ "segment_count_peak_delta": 1709,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260830T000343559627Z.pickle",
76
+ "timestamp_utc": "2026-08-30T00:03:43.800305Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000000_20260830T000343559627Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d10d50059a6f4bcacf3f9ed0f5bbaed5db49b1129ca1c4b452da57c04966c9d5
3
+ size 12137648
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260830T000344599451Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 69838439936,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 91028974592,
5
+ "active_bytes_peak_delta": 21190531584,
6
+ "after": {
7
+ "active_bytes_current": 69838439936,
8
+ "active_bytes_peak": 91028974592,
9
+ "allocated_bytes_current": 69838439936,
10
+ "allocated_bytes_peak": 91028974592,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 1743649280,
13
+ "inactive_split_bytes_peak": 13251217920,
14
+ "iteration": 1,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 93361012736,
21
+ "reserved_bytes_peak": 93361012736,
22
+ "segment_count_current": 2767,
23
+ "segment_count_peak": 2767,
24
+ "timestamp_utc": "2026-08-30T00:03:44.599046Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 69838439936,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 91028974592,
30
+ "allocated_bytes_peak_delta": 21190531584,
31
+ "before": {
32
+ "active_bytes_current": 69838439936,
33
+ "active_bytes_peak": 69838443008,
34
+ "allocated_bytes_current": 69838439936,
35
+ "allocated_bytes_peak": 69838443008,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 1743649280,
38
+ "inactive_split_bytes_peak": 13251217920,
39
+ "iteration": 1,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 71605157888,
46
+ "reserved_bytes_peak": 71605157888,
47
+ "segment_count_current": 2597,
48
+ "segment_count_peak": 2597,
49
+ "timestamp_utc": "2026-08-30T00:03:43.836519Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1743649280,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 13251217920,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 1,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 93361012736,
68
+ "reserved_bytes_current_delta": 21755854848,
69
+ "reserved_bytes_peak": 93361012736,
70
+ "reserved_bytes_peak_delta": 21755854848,
71
+ "segment_count_current": 2767,
72
+ "segment_count_current_delta": 170,
73
+ "segment_count_peak": 2767,
74
+ "segment_count_peak_delta": 170,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260830T000344599451Z.pickle",
76
+ "timestamp_utc": "2026-08-30T00:03:44.716971Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000001_20260830T000344599451Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0898e3930797946afe6c920f1fe1a1cabfaad1b752d333a537b77bd66e6bb99d
3
+ size 4795970
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260830T000345380733Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 69838439936,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 91037882368,
5
+ "active_bytes_peak_delta": 8907776,
6
+ "after": {
7
+ "active_bytes_current": 69838439936,
8
+ "active_bytes_peak": 91037882368,
9
+ "allocated_bytes_current": 69838439936,
10
+ "allocated_bytes_peak": 91037882368,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 1743649280,
13
+ "inactive_split_bytes_peak": 13251217920,
14
+ "iteration": 2,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 93361012736,
21
+ "reserved_bytes_peak": 93361012736,
22
+ "segment_count_current": 2767,
23
+ "segment_count_peak": 2767,
24
+ "timestamp_utc": "2026-08-30T00:03:45.380326Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 69838439936,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 91037882368,
30
+ "allocated_bytes_peak_delta": 8907776,
31
+ "before": {
32
+ "active_bytes_current": 69838439936,
33
+ "active_bytes_peak": 91028974592,
34
+ "allocated_bytes_current": 69838439936,
35
+ "allocated_bytes_peak": 91028974592,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 1743649280,
38
+ "inactive_split_bytes_peak": 13251217920,
39
+ "iteration": 2,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 93361012736,
46
+ "reserved_bytes_peak": 93361012736,
47
+ "segment_count_current": 2767,
48
+ "segment_count_peak": 2767,
49
+ "timestamp_utc": "2026-08-30T00:03:44.725760Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1743649280,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 13251217920,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 2,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 93361012736,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 93361012736,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 2767,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2767,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260830T000345380733Z.pickle",
76
+ "timestamp_utc": "2026-08-30T00:03:45.486125Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000002_20260830T000345380733Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:880339e9db0617788e75c2e22f2a0546c808b71b61f90175f39f760cd6a65596
3
+ size 5312337
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260830T000347043102Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 69855483392,
3
+ "active_bytes_current_delta": 17043456,
4
+ "active_bytes_peak": 91050207232,
5
+ "active_bytes_peak_delta": 12324864,
6
+ "after": {
7
+ "active_bytes_current": 69855483392,
8
+ "active_bytes_peak": 91050207232,
9
+ "allocated_bytes_current": 69855483392,
10
+ "allocated_bytes_peak": 91050207232,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 1936321024,
13
+ "inactive_split_bytes_peak": 13251217920,
14
+ "iteration": 3,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 93562339328,
21
+ "reserved_bytes_peak": 93562339328,
22
+ "segment_count_current": 2850,
23
+ "segment_count_peak": 2850,
24
+ "timestamp_utc": "2026-08-30T00:03:47.042627Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 69855483392,
28
+ "allocated_bytes_current_delta": 17043456,
29
+ "allocated_bytes_peak": 91050207232,
30
+ "allocated_bytes_peak_delta": 12324864,
31
+ "before": {
32
+ "active_bytes_current": 69838439936,
33
+ "active_bytes_peak": 91037882368,
34
+ "allocated_bytes_current": 69838439936,
35
+ "allocated_bytes_peak": 91037882368,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 1743649280,
38
+ "inactive_split_bytes_peak": 13251217920,
39
+ "iteration": 3,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 93361012736,
46
+ "reserved_bytes_peak": 93361012736,
47
+ "segment_count_current": 2767,
48
+ "segment_count_peak": 2767,
49
+ "timestamp_utc": "2026-08-30T00:03:45.494095Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1936321024,
55
+ "inactive_split_bytes_current_delta": 192671744,
56
+ "inactive_split_bytes_peak": 13251217920,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 3,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 93562339328,
68
+ "reserved_bytes_current_delta": 201326592,
69
+ "reserved_bytes_peak": 93562339328,
70
+ "reserved_bytes_peak_delta": 201326592,
71
+ "segment_count_current": 2850,
72
+ "segment_count_current_delta": 83,
73
+ "segment_count_peak": 2850,
74
+ "segment_count_peak_delta": 83,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260830T000347043102Z.pickle",
76
+ "timestamp_utc": "2026-08-30T00:03:47.152851Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000003_20260830T000347043102Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:549f39ba4b012ef7fb6672d576fd170f44c1b0f12300bfee9d0def15bd6c031a
3
+ size 5165025
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260830T000347538422Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 69855483392,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 91050207232,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 69855483392,
8
+ "active_bytes_peak": 91050207232,
9
+ "allocated_bytes_current": 69855483392,
10
+ "allocated_bytes_peak": 91050207232,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 1936321024,
13
+ "inactive_split_bytes_peak": 13251217920,
14
+ "iteration": 4,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 93562339328,
21
+ "reserved_bytes_peak": 93562339328,
22
+ "segment_count_current": 2850,
23
+ "segment_count_peak": 2850,
24
+ "timestamp_utc": "2026-08-30T00:03:47.538017Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 69855483392,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 91050207232,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 69855483392,
33
+ "active_bytes_peak": 91050207232,
34
+ "allocated_bytes_current": 69855483392,
35
+ "allocated_bytes_peak": 91050207232,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 1936321024,
38
+ "inactive_split_bytes_peak": 13251217920,
39
+ "iteration": 4,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 93562339328,
46
+ "reserved_bytes_peak": 93562339328,
47
+ "segment_count_current": 2850,
48
+ "segment_count_peak": 2850,
49
+ "timestamp_utc": "2026-08-30T00:03:47.161648Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1936321024,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 13251217920,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 4,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 93562339328,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 93562339328,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 2850,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2850,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260830T000347538422Z.pickle",
76
+ "timestamp_utc": "2026-08-30T00:03:47.598252Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000004_20260830T000347538422Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a15b96966d439e291061881c345ef0e5ab7a4849a5e74c2fd0687fac31e423e1
3
+ size 2603125
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260830T000347974022Z.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "active_bytes_current": 69855483392,
3
+ "active_bytes_current_delta": 0,
4
+ "active_bytes_peak": 91050207232,
5
+ "active_bytes_peak_delta": 0,
6
+ "after": {
7
+ "active_bytes_current": 69855483392,
8
+ "active_bytes_peak": 91050207232,
9
+ "allocated_bytes_current": 69855483392,
10
+ "allocated_bytes_peak": 91050207232,
11
+ "device": 0,
12
+ "inactive_split_bytes_current": 1936321024,
13
+ "inactive_split_bytes_peak": 13251217920,
14
+ "iteration": 5,
15
+ "mode": "iteration",
16
+ "num_alloc_retries": 0,
17
+ "num_ooms": 0,
18
+ "phase": "iteration_end",
19
+ "rank": 0,
20
+ "reserved_bytes_current": 93562339328,
21
+ "reserved_bytes_peak": 93562339328,
22
+ "segment_count_current": 2850,
23
+ "segment_count_peak": 2850,
24
+ "timestamp_utc": "2026-08-30T00:03:47.973627Z",
25
+ "world_size": 8
26
+ },
27
+ "allocated_bytes_current": 69855483392,
28
+ "allocated_bytes_current_delta": 0,
29
+ "allocated_bytes_peak": 91050207232,
30
+ "allocated_bytes_peak_delta": 0,
31
+ "before": {
32
+ "active_bytes_current": 69855483392,
33
+ "active_bytes_peak": 91050207232,
34
+ "allocated_bytes_current": 69855483392,
35
+ "allocated_bytes_peak": 91050207232,
36
+ "device": 0,
37
+ "inactive_split_bytes_current": 1936321024,
38
+ "inactive_split_bytes_peak": 13251217920,
39
+ "iteration": 5,
40
+ "mode": "iteration",
41
+ "num_alloc_retries": 0,
42
+ "num_ooms": 0,
43
+ "phase": "iteration_start",
44
+ "rank": 0,
45
+ "reserved_bytes_current": 93562339328,
46
+ "reserved_bytes_peak": 93562339328,
47
+ "segment_count_current": 2850,
48
+ "segment_count_peak": 2850,
49
+ "timestamp_utc": "2026-08-30T00:03:47.601021Z",
50
+ "world_size": 8
51
+ },
52
+ "device": 0,
53
+ "device_delta": 0,
54
+ "inactive_split_bytes_current": 1936321024,
55
+ "inactive_split_bytes_current_delta": 0,
56
+ "inactive_split_bytes_peak": 13251217920,
57
+ "inactive_split_bytes_peak_delta": 0,
58
+ "iteration": 5,
59
+ "iteration_delta": 0,
60
+ "mode": "iteration",
61
+ "num_alloc_retries": 0,
62
+ "num_alloc_retries_delta": 0,
63
+ "num_ooms": 0,
64
+ "num_ooms_delta": 0,
65
+ "rank": 0,
66
+ "rank_delta": 0,
67
+ "reserved_bytes_current": 93562339328,
68
+ "reserved_bytes_current_delta": 0,
69
+ "reserved_bytes_peak": 93562339328,
70
+ "reserved_bytes_peak_delta": 0,
71
+ "segment_count_current": 2850,
72
+ "segment_count_current_delta": 0,
73
+ "segment_count_peak": 2850,
74
+ "segment_count_peak_delta": 0,
75
+ "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260830T000347974022Z.pickle",
76
+ "timestamp_utc": "2026-08-30T00:03:48.030675Z",
77
+ "world_size": 8,
78
+ "world_size_delta": 0
79
+ }
tierb-fixed-moonep-full/moonep/tierbmem/tierbmem_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async__memiteration__it6/rep1/memory_snapshots/snapshot_iteration_rank0000_iter000005_20260830T000347974022Z.pickle ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0ef1dc6c55dfd3556d6ddd5f3e528b1317e53fc7a14b37a2a47b6261a37960eb
3
+ size 2603125