Hugging Face's logo

jalenluorion
/
intermediates

Model card Files
xet
 Community
jalenluorion commited on
Commit
bb6dfd4
·
verified ·
1 Parent(s): 3843a15

Add files using upload-large-folder tool

Browse files
Files changed (14) hide show
  1. in-progress/t3-archive/orchard-flame-15/gate_b_tier/MANIFEST.json +103 -0
  2. in-progress/t3-archive/orchard-flame-15/gate_b_tier/README.md +11 -0
  3. in-progress/t3-archive/orchard-flame-15/gate_b_tier/SHA256SUMS +11 -0
  4. in-progress/t3-archive/orchard-flame-15/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank0.pt +3 -0
  5. in-progress/t3-archive/orchard-flame-15/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank1.pt +3 -0
  6. in-progress/t3-archive/orchard-flame-15/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank2.pt +3 -0
  7. in-progress/t3-archive/orchard-flame-15/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank3.pt +3 -0
  8. in-progress/t3-archive/orchard-flame-15/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank4.pt +3 -0
  9. in-progress/t3-archive/orchard-flame-15/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank5.pt +3 -0
  10. in-progress/t3-archive/orchard-flame-15/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank6.pt +3 -0
  11. in-progress/t3-archive/orchard-flame-15/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank7.pt +3 -0
  12. in-progress/t3-archive/orchard-flame-15/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-trunk.pt +3 -0
  13. in-progress/t3-archive/orchard-flame-15/gate_b_tier/gate_b_tier.log +101 -0
  14. in-progress/t3-archive/orchard-flame-15/gate_b_tier/result_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.json +4044 -0
in-progress/t3-archive/orchard-flame-15/gate_b_tier/MANIFEST.json ADDED
@@ -0,0 +1,103 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "folder": "gate_b_tier",
3
+ "hub_path": "in-progress/t3-archive/orchard-flame-15/gate_b_tier",
4
+ "node": "orchard-flame-15",
5
+ "tier": 8,
6
+ "arm": "tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn",
7
+ "src_dir": "/tmp/jalenl/dev/gate_b_tier",
8
+ "form_on_node": "EP shards",
9
+ "merged_note": "no merged form was made",
10
+ "notes": "EP-trainer gate run gate_b_tier (full 1e9)",
11
+ "result": {
12
+ "final_eval_on_loss": {
13
+ "main_heldout": 2.839003,
14
+ "xd_lang": 5.74657,
15
+ "xd_code": 3.575041
16
+ },
17
+ "non_embed": 3240825984,
18
+ "steps": 7629,
19
+ "world": 8
20
+ },
21
+ "created": "2026-10-30 19:15:12 EDT",
22
+ "files": [
23
+ {
24
+ "name": "ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank0.pt",
25
+ "kind": "ckpt_ep_shard",
26
+ "src": "/tmp/jalenl/dev/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank0.pt",
27
+ "bytes": 753798370,
28
+ "sha256": "578692d35e400e61ee94349b4316b7c8164b01570002102b3ded790b5c11d56f"
29
+ },
30
+ {
31
+ "name": "ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank1.pt",
32
+ "kind": "ckpt_ep_shard",
33
+ "src": "/tmp/jalenl/dev/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank1.pt",
34
+ "bytes": 753798370,
35
+ "sha256": "b74b70934270af87e8f2468fbbf30bb604092b343d5761f4bac97b59e64e1d80"
36
+ },
37
+ {
38
+ "name": "ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank2.pt",
39
+ "kind": "ckpt_ep_shard",
40
+ "src": "/tmp/jalenl/dev/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank2.pt",
41
+ "bytes": 753798370,
42
+ "sha256": "6ada416115a815cdb5b947708ada28f9b756db802ef49d13603142bcc822cb42"
43
+ },
44
+ {
45
+ "name": "ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank3.pt",
46
+ "kind": "ckpt_ep_shard",
47
+ "src": "/tmp/jalenl/dev/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank3.pt",
48
+ "bytes": 753798370,
49
+ "sha256": "51ab97563ccd8fe916048f0024b200c01a87ef8e1a522393e44bc28437cd6f0b"
50
+ },
51
+ {
52
+ "name": "ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank4.pt",
53
+ "kind": "ckpt_ep_shard",
54
+ "src": "/tmp/jalenl/dev/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank4.pt",
55
+ "bytes": 753798370,
56
+ "sha256": "5ad9cad7b60ba70780cfa98328a4800ffcb9ada6025f4656d1b43bb10204261e"
57
+ },
58
+ {
59
+ "name": "ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank5.pt",
60
+ "kind": "ckpt_ep_shard",
61
+ "src": "/tmp/jalenl/dev/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank5.pt",
62
+ "bytes": 753798370,
63
+ "sha256": "099e0662696b85bf7879f3e10110c59776d82aa4bd6f22c9d95d35d09ac142cd"
64
+ },
65
+ {
66
+ "name": "ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank6.pt",
67
+ "kind": "ckpt_ep_shard",
68
+ "src": "/tmp/jalenl/dev/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank6.pt",
69
+ "bytes": 753798370,
70
+ "sha256": "c6df1195ccde66d1a0191fee945bcc91292fe75396971035bc45065393d19798"
71
+ },
72
+ {
73
+ "name": "ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank7.pt",
74
+ "kind": "ckpt_ep_shard",
75
+ "src": "/tmp/jalenl/dev/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank7.pt",
76
+ "bytes": 753798370,
77
+ "sha256": "e80ac8cfeaa352637de409755739569541de2a3ca032e8df02d2b914f2652d2d"
78
+ },
79
+ {
80
+ "name": "ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-trunk.pt",
81
+ "kind": "ckpt_ep_shard",
82
+ "src": "/tmp/jalenl/dev/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-trunk.pt",
83
+ "bytes": 525077082,
84
+ "sha256": "de3c0c841bd5cfead6b2b478b5a3aa82b19c23730f50daf2595f05939178a1e7"
85
+ },
86
+ {
87
+ "name": "result_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.json",
88
+ "kind": "result",
89
+ "src": "/tmp/jalenl/dev/gate_b_tier/result_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.json",
90
+ "bytes": 76529,
91
+ "sha256": "c0c333e1c881894130a7d28c92e923364b7298407d88000057e280eecef01f8e"
92
+ },
93
+ {
94
+ "name": "gate_b_tier.log",
95
+ "kind": "train_log",
96
+ "src": "/home/jalenl/dm_out/dev/gate_b_tier.log",
97
+ "bytes": 47528,
98
+ "sha256": "a8cae8954863992ba4f0d74a7406dc3c3e151e72c46fca2234768601767f964b"
99
+ }
100
+ ],
101
+ "withheld": [],
102
+ "total_bytes": 6555588099
103
+ }
in-progress/t3-archive/orchard-flame-15/gate_b_tier/README.md ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # gate_b_tier
2
+
3
+ Archived intermediate files from node-local storage (orchard-flame-15), 2026-10-30.
4
+
5
+ - run: `tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn`
6
+ - what: EP-trainer gate run gate_b_tier (full 1e9)
7
+ - checkpoint form: EP shards; no merged form was made
8
+ - held-out loss at the end of training: {'main_heldout': 2.839003, 'xd_lang': 5.74657, 'xd_code': 3.575041}
9
+
10
+ File sizes, sources and sha256 are in `MANIFEST.json` and `SHA256SUMS`.
11
+
in-progress/t3-archive/orchard-flame-15/gate_b_tier/SHA256SUMS ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 578692d35e400e61ee94349b4316b7c8164b01570002102b3ded790b5c11d56f ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank0.pt
2
+ b74b70934270af87e8f2468fbbf30bb604092b343d5761f4bac97b59e64e1d80 ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank1.pt
3
+ 6ada416115a815cdb5b947708ada28f9b756db802ef49d13603142bcc822cb42 ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank2.pt
4
+ 51ab97563ccd8fe916048f0024b200c01a87ef8e1a522393e44bc28437cd6f0b ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank3.pt
5
+ 5ad9cad7b60ba70780cfa98328a4800ffcb9ada6025f4656d1b43bb10204261e ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank4.pt
6
+ 099e0662696b85bf7879f3e10110c59776d82aa4bd6f22c9d95d35d09ac142cd ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank5.pt
7
+ c6df1195ccde66d1a0191fee945bcc91292fe75396971035bc45065393d19798 ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank6.pt
8
+ e80ac8cfeaa352637de409755739569541de2a3ca032e8df02d2b914f2652d2d ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank7.pt
9
+ de3c0c841bd5cfead6b2b478b5a3aa82b19c23730f50daf2595f05939178a1e7 ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-trunk.pt
10
+ c0c333e1c881894130a7d28c92e923364b7298407d88000057e280eecef01f8e result_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.json
11
+ a8cae8954863992ba4f0d74a7406dc3c3e151e72c46fca2234768601767f964b gate_b_tier.log
in-progress/t3-archive/orchard-flame-15/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:578692d35e400e61ee94349b4316b7c8164b01570002102b3ded790b5c11d56f
3
+ size 753798370
in-progress/t3-archive/orchard-flame-15/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b74b70934270af87e8f2468fbbf30bb604092b343d5761f4bac97b59e64e1d80
3
+ size 753798370
in-progress/t3-archive/orchard-flame-15/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6ada416115a815cdb5b947708ada28f9b756db802ef49d13603142bcc822cb42
3
+ size 753798370
in-progress/t3-archive/orchard-flame-15/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:51ab97563ccd8fe916048f0024b200c01a87ef8e1a522393e44bc28437cd6f0b
3
+ size 753798370
in-progress/t3-archive/orchard-flame-15/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank4.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5ad9cad7b60ba70780cfa98328a4800ffcb9ada6025f4656d1b43bb10204261e
3
+ size 753798370
in-progress/t3-archive/orchard-flame-15/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank5.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:099e0662696b85bf7879f3e10110c59776d82aa4bd6f22c9d95d35d09ac142cd
3
+ size 753798370
in-progress/t3-archive/orchard-flame-15/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank6.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c6df1195ccde66d1a0191fee945bcc91292fe75396971035bc45065393d19798
3
+ size 753798370
in-progress/t3-archive/orchard-flame-15/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-rank7.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e80ac8cfeaa352637de409755739569541de2a3ca032e8df02d2b914f2652d2d
3
+ size 753798370
in-progress/t3-archive/orchard-flame-15/gate_b_tier/ckpt_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.ep-trunk.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:de3c0c841bd5cfead6b2b478b5a3aa82b19c23730f50daf2595f05939178a1e7
3
+ size 525077082
in-progress/t3-archive/orchard-flame-15/gate_b_tier/gate_b_tier.log ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ START Sat Oct 24 01:40:08 EDT 2026 host orchard-flame-15 out /tmp/jalenl/dev/gate_b_tier args --arm 8 --mult 1 --root /tmp/jalenl/v4 --store-norm --gran 4 --shared 1 --router learned --cap-factor 0 --ep --micro-batch 4 --accum 2 --tier-gather
2
+ WARNING:__main__:
3
+ *****************************************
4
+ Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
5
+ *****************************************
6
+ [W1024 01:40:15.497462681 CUDAAllocatorConfig.h:28] Warning: expandable_segments not supported on this platform (function operator())
7
+ [W1024 01:40:15.709300646 CUDAAllocatorConfig.h:28] Warning: expandable_segments not supported on this platform (function operator())
8
+ [W1024 01:40:15.881238137 CUDAAllocatorConfig.h:28] Warning: expandable_segments not supported on this platform (function operator())
9
+ [W1024 01:40:16.172467639 CUDAAllocatorConfig.h:28] Warning: expandable_segments not supported on this platform (function operator())
10
+ [W1024 01:40:16.213561834 CUDAAllocatorConfig.h:28] Warning: expandable_segments not supported on this platform (function operator())
11
+ [W1024 01:40:16.265184370 CUDAAllocatorConfig.h:28] Warning: expandable_segments not supported on this platform (function operator())
12
+ [W1024 01:40:16.348517559 CUDAAllocatorConfig.h:28] Warning: expandable_segments not supported on this platform (function operator())
13
+ [W1024 01:40:16.348619479 CUDAAllocatorConfig.h:28] Warning: expandable_segments not supported on this platform (function operator())
14
+ {"arm": "tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn", "steps": 7629, "train_seqs": 488281, "world": 8, "tokens": 999948288, "moe_layers": [3, 9], "non_embed_M": 3240.8, "added_M": 3021.4, "tokens_freeze": "3cb3d04a50816f7a", "active_M": 230.1, "total_M": 3240.8, "embed_M": 36.9, "d_ff_e": 853, "expert_width": 213, "experts_per_layer": 2048, "k_active": 3, "n_shared": 1, "units_per_layer": 2048, "unit_params": 736128, "seq_ids": {"step0": [56082, 419616], "step1": [287328, 497789], "sha256_steps01": "f55a693087f3af64"}, "ep": 8, "experts_per_rank": 256, "experts_per_rank_all": [256, 256, 256, 256, 256, 256, 256, 256], "ckpt_experts": false, "zero1": false, "fast_dispatch": true, "bf16_dispatch": false, "tier_gather": true}
15
+ {"step": 0, "loss": 10.603790283203125, "lr": 2e-06, "elapsed_s": 14.28202748298645, "util": [0.7944, 0.5996], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9078369140625, "reads_keep3": 2.7408447265625}, {"k": 3, "reads_keep1": 2.788818359375, "reads_keep3": 2.468994140625}], "step_s": 14.2272, "grad_norm": 11.13349, "grad_norm_mean": 11.13349, "clip_frac": 1.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [16.87, 16.88, 16.87, 16.88, 16.89, 16.91, 16.9, 16.89], "max_mem_reserved_gb": [17.22, 17.24, 17.2, 17.22, 17.26, 17.26, 17.26, 17.26]}
16
+ {"PEAK_MEM_STEP100": "tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn", "step_s": 0.7733, "grad_norm": 0.90461, "grad_norm_mean": 2.08934, "clip_frac": 0.83, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.69, 21.68, 21.68, 21.71], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.04, 22.02, 22.04, 22.07]}
17
+ {"step": 100, "loss": 5.4917120933532715, "lr": 0.00020199999999999998, "elapsed_s": 91.6028208732605, "util": [0.9331, 0.8555], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.979248046875, "reads_keep3": 2.91796875}, {"k": 3, "reads_keep1": 2.9642333984375, "reads_keep3": 2.8978271484375}], "step_s": 0.7733, "grad_norm": 0.90461, "grad_norm_mean": 2.08934, "clip_frac": 0.83, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.69, 21.68, 21.68, 21.71], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.04, 22.02, 22.04, 22.07]}
18
+ {"step": 200, "loss": 4.816793918609619, "lr": 0.000402, "elapsed_s": 167.11967039108276, "util": [0.9146, 0.8867], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9893798828125, "reads_keep3": 2.941650390625}, {"k": 3, "reads_keep1": 2.9835205078125, "reads_keep3": 2.941162109375}], "step_s": 0.7553, "grad_norm": 0.66178, "grad_norm_mean": 0.89119, "clip_frac": 0.19, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.68, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.02, 22.04, 22.09]}
19
+ {"step": 300, "loss": 4.762606620788574, "lr": 0.0006, "elapsed_s": 242.11455154418945, "util": [0.877, 0.8628], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.970458984375, "reads_keep3": 2.91796875}, {"k": 3, "reads_keep1": 2.9696044921875, "reads_keep3": 2.913818359375}], "step_s": 0.75, "grad_norm": 1.28636, "grad_norm_mean": 2.65395, "clip_frac": 0.49, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.04, 22.09]}
20
+ {"step": 400, "loss": 4.119720458984375, "lr": 0.0005997519852153506, "elapsed_s": 317.84852719306946, "util": [0.9248, 0.9292], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9630126953125, "reads_keep3": 2.9049072265625}, {"k": 3, "reads_keep1": 2.957763671875, "reads_keep3": 2.9110107421875}], "step_s": 0.7573, "grad_norm": 0.42827, "grad_norm_mean": 0.64226, "clip_frac": 0.08, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.04, 22.09]}
21
+ {"step": 500, "loss": 3.9423646926879883, "lr": 0.0005990083965009096, "elapsed_s": 392.6189374923706, "util": [0.9316, 0.9556], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.975830078125, "reads_keep3": 2.9346923828125}, {"k": 3, "reads_keep1": 2.9722900390625, "reads_keep3": 2.938720703125}], "step_s": 0.7477, "grad_norm": 0.32777, "grad_norm_mean": 0.64873, "clip_frac": 0.13, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
22
+ {"step": 600, "loss": 3.8606295585632324, "lr": 0.0005977705999381204, "elapsed_s": 467.8239600658417, "util": [0.8862, 0.9526], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.98681640625, "reads_keep3": 2.9281005859375}, {"k": 3, "reads_keep1": 2.9840087890625, "reads_keep3": 2.9398193359375}], "step_s": 0.752, "grad_norm": 1.06046, "grad_norm_mean": 0.49575, "clip_frac": 0.07, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
23
+ {"step": 700, "loss": 3.6110830307006836, "lr": 0.0005960408695406716, "elapsed_s": 541.3506631851196, "util": [0.8955, 0.9541], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9793701171875, "reads_keep3": 2.9197998046875}, {"k": 3, "reads_keep1": 2.97119140625, "reads_keep3": 2.92822265625}], "step_s": 0.7353, "grad_norm": 2.27035, "grad_norm_mean": 0.59618, "clip_frac": 0.13, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
24
+ {"step": 800, "loss": 3.4680235385894775, "lr": 0.0005938223830768002, "elapsed_s": 615.0662519931793, "util": [0.8975, 0.9688], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9862060546875, "reads_keep3": 2.9453125}, {"k": 3, "reads_keep1": 2.978515625, "reads_keep3": 2.9522705078125}], "step_s": 0.7371, "grad_norm": 0.69486, "grad_norm_mean": 0.42549, "clip_frac": 0.05, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
25
+ {"step": 900, "loss": 3.5275583267211914, "lr": 0.0005911192162312665, "elapsed_s": 687.6100196838379, "util": [0.917, 0.9854], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9888916015625, "reads_keep3": 2.953369140625}, {"k": 3, "reads_keep1": 2.9837646484375, "reads_keep3": 2.9569091796875}], "step_s": 0.7255, "grad_norm": 0.25159, "grad_norm_mean": 0.28878, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
26
+ {"TRUNK_CHECKSUM": 1000, "sum": 14014.94505915981, "abs": 4561703.433294149}
27
+ {"step": 1000, "loss": 3.5475330352783203, "lr": 0.0005879363351177225, "elapsed_s": 761.1710834503174, "util": [0.9072, 0.9805], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.980224609375, "reads_keep3": 2.9296875}, {"k": 3, "reads_keep1": 2.9739990234375, "reads_keep3": 2.939208984375}], "step_s": 0.7355, "grad_norm": 0.25038, "grad_norm_mean": 0.30107, "clip_frac": 0.02, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
28
+ {"step": 1100, "loss": 3.6279101371765137, "lr": 0.0005842795871552345, "elapsed_s": 834.9443986415863, "util": [0.9219, 0.9858], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9876708984375, "reads_keep3": 2.950439453125}, {"k": 3, "reads_keep1": 2.98291015625, "reads_keep3": 2.954833984375}], "step_s": 0.7378, "grad_norm": 0.24847, "grad_norm_mean": 0.25598, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
29
+ {"step": 1200, "loss": 3.535391092300415, "lr": 0.0005801556903257169, "elapsed_s": 909.2344558238983, "util": [0.9189, 0.981], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9906005859375, "reads_keep3": 2.956787109375}, {"k": 3, "reads_keep1": 2.98095703125, "reads_keep3": 2.9573974609375}], "step_s": 0.7429, "grad_norm": 0.23094, "grad_norm_mean": 0.24032, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
30
+ {"step": 1300, "loss": 3.3749685287475586, "lr": 0.0005755722208320148, "elapsed_s": 984.4957759380341, "util": [0.9287, 0.9917], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9925537109375, "reads_keep3": 2.95458984375}, {"k": 3, "reads_keep1": 2.98291015625, "reads_keep3": 2.9512939453125}], "step_s": 0.7526, "grad_norm": 0.2391, "grad_norm_mean": 0.25197, "clip_frac": 0.01, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
31
+ {"step": 1400, "loss": 3.142401695251465, "lr": 0.000570537599179309, "elapsed_s": 1059.1784732341766, "util": [0.9165, 0.9897], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.981689453125, "reads_keep3": 2.920166015625}, {"k": 3, "reads_keep1": 2.9710693359375, "reads_keep3": 2.91748046875}], "step_s": 0.7468, "grad_norm": 0.2181, "grad_norm_mean": 0.22604, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
32
+ {"step": 1500, "loss": 3.2960891723632812, "lr": 0.0005650610747054142, "elapsed_s": 1134.9925062656403, "util": [0.9243, 0.9893], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.991943359375, "reads_keep3": 2.940185546875}, {"k": 3, "reads_keep1": 2.982177734375, "reads_keep3": 2.9522705078125}], "step_s": 0.7582, "grad_norm": 0.20629, "grad_norm_mean": 0.22417, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
33
+ {"step": 1600, "loss": 3.2592875957489014, "lr": 0.0005591527085883902, "elapsed_s": 1209.1713495254517, "util": [0.9263, 0.9888], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9864501953125, "reads_keep3": 2.9544677734375}, {"k": 3, "reads_keep1": 2.9810791015625, "reads_keep3": 2.9556884765625}], "step_s": 0.7417, "grad_norm": 0.23391, "grad_norm_mean": 0.24642, "clip_frac": 0.01, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
34
+ {"step": 1700, "loss": 3.318453311920166, "lr": 0.000552823355362682, "elapsed_s": 1282.6599502563477, "util": [0.9297, 0.9917], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9873046875, "reads_keep3": 2.9434814453125}, {"k": 3, "reads_keep1": 2.972900390625, "reads_keep3": 2.9423828125}], "step_s": 0.7349, "grad_norm": 0.20709, "grad_norm_mean": 0.2237, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
35
+ {"step": 1800, "loss": 3.2907724380493164, "lr": 0.0005460846429777509, "elapsed_s": 1358.0658473968506, "util": [0.9307, 0.9897], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9906005859375, "reads_keep3": 2.96044921875}, {"k": 3, "reads_keep1": 2.981689453125, "reads_keep3": 2.95849609375}], "step_s": 0.7542, "grad_norm": 0.21604, "grad_norm_mean": 0.23864, "clip_frac": 0.01, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
36
+ {"step": 1900, "loss": 3.3151278495788574, "lr": 0.0005389489514358262, "elapsed_s": 1431.3323302268982, "util": [0.9346, 0.9927], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9892578125, "reads_keep3": 2.9520263671875}, {"k": 3, "reads_keep1": 2.9786376953125, "reads_keep3": 2.9476318359375}], "step_s": 0.7325, "grad_norm": 0.20736, "grad_norm_mean": 0.21179, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
37
+ {"TRUNK_CHECKSUM": 2000, "sum": -6405.5529512878165, "abs": 5290266.195750939}
38
+ {"step": 2000, "loss": 3.2122890949249268, "lr": 0.0005314293900480271, "elapsed_s": 1506.8471064567566, "util": [0.9365, 0.9932], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.988037109375, "reads_keep3": 2.9285888671875}, {"k": 3, "reads_keep1": 2.974609375, "reads_keep3": 2.9283447265625}], "step_s": 0.7553, "grad_norm": 0.83733, "grad_norm_mean": 0.29064, "clip_frac": 0.02, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
39
+ {"step": 2100, "loss": 3.0985865592956543, "lr": 0.0005235397733506371, "elapsed_s": 1582.2666125297546, "util": [0.9316, 0.9912], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.990478515625, "reads_keep3": 2.9525146484375}, {"k": 3, "reads_keep1": 2.9775390625, "reads_keep3": 2.9490966796875}], "step_s": 0.7541, "grad_norm": 0.20209, "grad_norm_mean": 0.22126, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
40
+ {"step": 2200, "loss": 3.3588297367095947, "lr": 0.0005152945957257758, "elapsed_s": 1655.7793147563934, "util": [0.9297, 0.9927], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9903564453125, "reads_keep3": 2.9365234375}, {"k": 3, "reads_keep1": 2.976318359375, "reads_keep3": 2.937744140625}], "step_s": 0.7352, "grad_norm": 0.2092, "grad_norm_mean": 0.2108, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
41
+ {"step": 2300, "loss": 3.402806282043457, "lr": 0.0005067090047730952, "elapsed_s": 1730.8209352493286, "util": [0.9399, 0.9956], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9910888671875, "reads_keep3": 2.9541015625}, {"k": 3, "reads_keep1": 2.9781494140625, "reads_keep3": 2.9521484375}], "step_s": 0.7503, "grad_norm": 0.22093, "grad_norm_mean": 0.20946, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
42
+ {"step": 2400, "loss": 3.3327414989471436, "lr": 0.0004977987734814191, "elapsed_s": 1806.522236585617, "util": [0.9351, 0.9971], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.993408203125, "reads_keep3": 2.9530029296875}, {"k": 3, "reads_keep1": 2.97705078125, "reads_keep3": 2.9482421875}], "step_s": 0.757, "grad_norm": 0.22474, "grad_norm_mean": 0.21412, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
43
+ {"step": 2500, "loss": 2.9692678451538086, "lr": 0.0004885802712514513, "elapsed_s": 1882.090324163437, "util": [0.9385, 0.9941], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9903564453125, "reads_keep3": 2.953369140625}, {"k": 3, "reads_keep1": 2.978515625, "reads_keep3": 2.9488525390625}], "step_s": 0.7557, "grad_norm": 0.20245, "grad_norm_mean": 0.20889, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
44
+ {"step": 2600, "loss": 3.281794548034668, "lr": 0.0004790704338227868, "elapsed_s": 1956.4616079330444, "util": [0.9341, 0.9927], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.994873046875, "reads_keep3": 2.9566650390625}, {"k": 3, "reads_keep1": 2.97900390625, "reads_keep3": 2.9503173828125}], "step_s": 0.7437, "grad_norm": 0.20784, "grad_norm_mean": 0.20703, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
45
+ {"step": 2700, "loss": 2.984389305114746, "lr": 0.0004692867321604769, "elapsed_s": 2030.7483987808228, "util": [0.9453, 0.9956], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9844970703125, "reads_keep3": 2.9420166015625}, {"k": 3, "reads_keep1": 2.965087890625, "reads_keep3": 2.9307861328125}], "step_s": 0.7429, "grad_norm": 0.21195, "grad_norm_mean": 0.21187, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
46
+ {"step": 2800, "loss": 3.272160053253174, "lr": 0.0004592471403583057, "elapsed_s": 2104.6319546699524, "util": [0.936, 0.9941], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.993408203125, "reads_keep3": 2.9515380859375}, {"k": 3, "reads_keep1": 2.982177734375, "reads_keep3": 2.950927734375}], "step_s": 0.7389, "grad_norm": 0.20636, "grad_norm_mean": 0.21141, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
47
+ {"step": 2900, "loss": 3.1806206703186035, "lr": 0.00044897010261774577, "elapsed_s": 2178.3615646362305, "util": [0.9458, 0.9971], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9932861328125, "reads_keep3": 2.95751953125}, {"k": 3, "reads_keep1": 2.9788818359375, "reads_keep3": 2.9490966796875}], "step_s": 0.7372, "grad_norm": 0.19491, "grad_norm_mean": 0.20595, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
48
+ {"TRUNK_CHECKSUM": 3000, "sum": -11753.1114531014, "abs": 5698862.969396745}
49
+ {"step": 3000, "loss": 2.70481014251709, "lr": 0.00043847449936325653, "elapsed_s": 2253.2748243808746, "util": [0.9492, 0.9971], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9908447265625, "reads_keep3": 2.954345703125}, {"k": 3, "reads_keep1": 2.9769287109375, "reads_keep3": 2.947265625}], "step_s": 0.7492, "grad_norm": 0.19955, "grad_norm_mean": 0.20256, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
50
+ {"step": 3100, "loss": 2.853893756866455, "lr": 0.00042777961255617835, "elapsed_s": 2325.9886758327484, "util": [0.9351, 0.9951], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.989501953125, "reads_keep3": 2.9544677734375}, {"k": 3, "reads_keep1": 2.972900390625, "reads_keep3": 2.944580078125}], "step_s": 0.7271, "grad_norm": 0.20644, "grad_norm_mean": 0.20348, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
51
+ {"step": 3200, "loss": 3.1224350929260254, "lr": 0.0004169050902709431, "elapsed_s": 2399.321888923645, "util": [0.959, 0.9971], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.990478515625, "reads_keep3": 2.95849609375}, {"k": 3, "reads_keep1": 2.976318359375, "reads_keep3": 2.945556640625}], "step_s": 0.7335, "grad_norm": 0.20256, "grad_norm_mean": 0.20802, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
52
+ {"step": 3300, "loss": 3.4007363319396973, "lr": 0.0004058709105986821, "elapsed_s": 2474.8059916496277, "util": [0.9556, 0.9985], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.99267578125, "reads_keep3": 2.9609375}, {"k": 3, "reads_keep1": 2.981201171875, "reads_keep3": 2.9544677734375}], "step_s": 0.7547, "grad_norm": 0.21276, "grad_norm_mean": 0.20332, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
53
+ {"step": 3400, "loss": 3.32659912109375, "lr": 0.000394697344944545, "elapsed_s": 2549.775881290436, "util": [0.9478, 0.9946], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9892578125, "reads_keep3": 2.9425048828125}, {"k": 3, "reads_keep1": 2.974365234375, "reads_keep3": 2.9466552734375}], "step_s": 0.7497, "grad_norm": 0.20353, "grad_norm_mean": 0.20228, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
54
+ {"step": 3500, "loss": 2.9369020462036133, "lr": 0.0003834049207861575, "elapsed_s": 2623.84023809433, "util": [0.9585, 0.9937], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9920654296875, "reads_keep3": 2.9344482421875}, {"k": 3, "reads_keep1": 2.975341796875, "reads_keep3": 2.93359375}], "step_s": 0.7407, "grad_norm": 0.19558, "grad_norm_mean": 0.20182, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
55
+ {"step": 3600, "loss": 3.1533007621765137, "lr": 0.0003720143839616368, "elapsed_s": 2698.953490257263, "util": [0.96, 0.9961], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9942626953125, "reads_keep3": 2.9639892578125}, {"k": 3, "reads_keep1": 2.9796142578125, "reads_keep3": 2.9532470703125}], "step_s": 0.751, "grad_norm": 0.19066, "grad_norm_mean": 0.20188, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
56
+ {"step": 3700, "loss": 2.912728786468506, "lr": 0.0003605466605564464, "elapsed_s": 2773.2590370178223, "util": [0.9551, 0.999], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.99072265625, "reads_keep3": 2.959228515625}, {"k": 3, "reads_keep1": 2.96826171875, "reads_keep3": 2.93994140625}], "step_s": 0.7431, "grad_norm": 0.21287, "grad_norm_mean": 0.19817, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
57
+ {"step": 3800, "loss": 2.966920852661133, "lr": 0.0003490228184591104, "elapsed_s": 2847.961940050125, "util": [0.9492, 0.9971], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9892578125, "reads_keep3": 2.9473876953125}, {"k": 3, "reads_keep1": 2.96923828125, "reads_keep3": 2.9351806640625}], "step_s": 0.747, "grad_norm": 0.20476, "grad_norm_mean": 0.19961, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
58
+ {"step": 3900, "loss": 2.9563255310058594, "lr": 0.00033746402865641464, "elapsed_s": 2922.7830786705017, "util": [0.9678, 0.999], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9906005859375, "reads_keep3": 2.95068359375}, {"k": 3, "reads_keep1": 2.965087890625, "reads_keep3": 2.935546875}], "step_s": 0.7483, "grad_norm": 0.19311, "grad_norm_mean": 0.2005, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
59
+ {"TRUNK_CHECKSUM": 4000, "sum": -13793.70563157677, "abs": 5889625.839165749}
60
+ {"step": 4000, "loss": 2.939749240875244, "lr": 0.0003258915263392015, "elapsed_s": 2998.234353542328, "util": [0.96, 0.9971], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.990478515625, "reads_keep3": 2.9595947265625}, {"k": 3, "reads_keep1": 2.9766845703125, "reads_keep3": 2.9468994140625}], "step_s": 0.7545, "grad_norm": 0.19522, "grad_norm_mean": 0.20158, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
61
+ {"step": 4100, "loss": 2.956996440887451, "lr": 0.00031432657189021235, "elapsed_s": 3073.1621329784393, "util": [0.9707, 0.9985], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9903564453125, "reads_keep3": 2.95654296875}, {"k": 3, "reads_keep1": 2.97265625, "reads_keep3": 2.9456787109375}], "step_s": 0.7493, "grad_norm": 0.19031, "grad_norm_mean": 0.20092, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
62
+ {"step": 4200, "loss": 2.7563636302948, "lr": 0.00030279041182564807, "elapsed_s": 3149.9751269817352, "util": [0.9487, 0.9917], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.994384765625, "reads_keep3": 2.95458984375}, {"k": 3, "reads_keep1": 2.9769287109375, "reads_keep3": 2.950439453125}], "step_s": 0.7681, "grad_norm": 0.19292, "grad_norm_mean": 0.20121, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
63
+ {"step": 4300, "loss": 2.962275981903076, "lr": 0.0002913042397622058, "elapsed_s": 3225.034797668457, "util": [0.9727, 0.9976], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.99072265625, "reads_keep3": 2.9556884765625}, {"k": 3, "reads_keep1": 2.97509765625, "reads_keep3": 2.9483642578125}], "step_s": 0.7506, "grad_norm": 0.19897, "grad_norm_mean": 0.19914, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
64
+ {"step": 4400, "loss": 2.7788233757019043, "lr": 0.00027988915748129826, "elapsed_s": 3301.5959877967834, "util": [0.957, 0.9937], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.987548828125, "reads_keep3": 2.9591064453125}, {"k": 3, "reads_keep1": 2.96875, "reads_keep3": 2.9462890625}], "step_s": 0.7656, "grad_norm": 0.20864, "grad_norm_mean": 0.20094, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
65
+ {"step": 4500, "loss": 3.030665874481201, "lr": 0.0002685661361619898, "elapsed_s": 3378.687537908554, "util": [0.96, 0.9976], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.99462890625, "reads_keep3": 2.9603271484375}, {"k": 3, "reads_keep1": 2.97509765625, "reads_keep3": 2.94580078125}], "step_s": 0.771, "grad_norm": 0.20155, "grad_norm_mean": 0.20009, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
66
+ {"step": 4600, "loss": 2.576869010925293, "lr": 0.0002573559778538662, "elapsed_s": 3455.1875426769257, "util": [0.9663, 0.9946], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9891357421875, "reads_keep3": 2.9534912109375}, {"k": 3, "reads_keep1": 2.9708251953125, "reads_keep3": 2.9390869140625}], "step_s": 0.765, "grad_norm": 0.19646, "grad_norm_mean": 0.20054, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
67
+ {"step": 4700, "loss": 2.840919256210327, "lr": 0.0002462792772606214, "elapsed_s": 3532.6897184848785, "util": [0.9648, 0.9927], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.99462890625, "reads_keep3": 2.9674072265625}, {"k": 3, "reads_keep1": 2.973388671875, "reads_keep3": 2.94580078125}], "step_s": 0.775, "grad_norm": 0.20917, "grad_norm_mean": 0.19826, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
68
+ {"step": 4800, "loss": 2.814150810241699, "lr": 0.00023535638390456822, "elapsed_s": 3611.532378435135, "util": [0.9707, 0.999], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.995849609375, "reads_keep3": 2.9617919921875}, {"k": 3, "reads_keep1": 2.972900390625, "reads_keep3": 2.9423828125}], "step_s": 0.7884, "grad_norm": 0.1892, "grad_norm_mean": 0.19727, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
69
+ {"step": 4900, "loss": 2.9387612342834473, "lr": 0.0002246073647415842, "elapsed_s": 3690.9922754764557, "util": [0.9639, 0.9937], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9947509765625, "reads_keep3": 2.95654296875}, {"k": 3, "reads_keep1": 2.9759521484375, "reads_keep3": 2.9425048828125}], "step_s": 0.7947, "grad_norm": 0.19499, "grad_norm_mean": 0.19865, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
70
+ {"TRUNK_CHECKSUM": 5000, "sum": -19057.451231015024, "abs": 5936931.873429785}
71
+ {"step": 5000, "loss": 3.032792568206787, "lr": 0.00021405196729517236, "elapsed_s": 3770.0510654449463, "util": [0.9751, 0.9995], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9921875, "reads_keep3": 2.9609375}, {"k": 3, "reads_keep1": 2.975830078125, "reads_keep3": 2.9495849609375}], "step_s": 0.7906, "grad_norm": 0.20141, "grad_norm_mean": 0.19843, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
72
+ {"step": 5100, "loss": 2.979060173034668, "lr": 0.00020370958337736627, "elapsed_s": 3849.664798259735, "util": [0.978, 0.9976], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.990478515625, "reads_keep3": 2.9471435546875}, {"k": 3, "reads_keep1": 2.974609375, "reads_keep3": 2.9388427734375}], "step_s": 0.7962, "grad_norm": 0.18802, "grad_norm_mean": 0.19778, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
73
+ {"step": 5200, "loss": 3.0355381965637207, "lr": 0.00019359921346313, "elapsed_s": 3929.0365781784058, "util": [0.9697, 0.9976], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.99365234375, "reads_keep3": 2.967529296875}, {"k": 3, "reads_keep1": 2.9744873046875, "reads_keep3": 2.9498291015625}], "step_s": 0.7937, "grad_norm": 0.19806, "grad_norm_mean": 0.19832, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
74
+ {"step": 5300, "loss": 3.0029296875, "lr": 0.00018373943178370066, "elapsed_s": 4008.2700369358063, "util": [0.9819, 1.0], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.99365234375, "reads_keep3": 2.970947265625}, {"k": 3, "reads_keep1": 2.9755859375, "reads_keep3": 2.9503173828125}], "step_s": 0.7923, "grad_norm": 0.19581, "grad_norm_mean": 0.19797, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
75
+ {"step": 5400, "loss": 3.0728814601898193, "lr": 0.00017414835220300347, "elapsed_s": 4088.0783507823944, "util": [0.9766, 0.9966], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.99462890625, "reads_keep3": 2.970458984375}, {"k": 3, "reads_keep1": 2.97119140625, "reads_keep3": 2.9466552734375}], "step_s": 0.7981, "grad_norm": 0.19208, "grad_norm_mean": 0.1978, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
76
+ {"step": 5500, "loss": 2.8215982913970947, "lr": 0.0001648435949398293, "elapsed_s": 4171.449336767197, "util": [0.9775, 0.998], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9835205078125, "reads_keep3": 2.92138671875}, {"k": 3, "reads_keep1": 2.953857421875, "reads_keep3": 2.9010009765625}], "step_s": 0.8337, "grad_norm": 0.20284, "grad_norm_mean": 0.19793, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
77
+ {"step": 5600, "loss": 2.9599080085754395, "lr": 0.00015584225419691021, "elapsed_s": 4253.901607513428, "util": [0.98, 0.9961], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9940185546875, "reads_keep3": 2.9664306640625}, {"k": 3, "reads_keep1": 2.97705078125, "reads_keep3": 2.9482421875}], "step_s": 0.8245, "grad_norm": 0.21032, "grad_norm_mean": 0.20075, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
78
+ {"step": 5700, "loss": 3.054381847381592, "lr": 0.00014716086675636405, "elapsed_s": 4333.841708660126, "util": [0.9834, 0.999], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.990966796875, "reads_keep3": 2.9639892578125}, {"k": 3, "reads_keep1": 2.9715576171875, "reads_keep3": 2.942626953125}], "step_s": 0.7994, "grad_norm": 0.19443, "grad_norm_mean": 0.1994, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
79
+ {"step": 5800, "loss": 2.800983428955078, "lr": 0.0001388153815992021, "elapsed_s": 4414.956063985825, "util": [0.9829, 0.998], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9940185546875, "reads_keep3": 2.9686279296875}, {"k": 3, "reads_keep1": 2.977294921875, "reads_keep3": 2.9537353515625}], "step_s": 0.8112, "grad_norm": 0.20158, "grad_norm_mean": 0.19817, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
80
+ {"step": 5900, "loss": 3.066985607147217, "lr": 0.00013082113060471372, "elapsed_s": 4496.873050928116, "util": [0.9834, 0.9961], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9921875, "reads_keep3": 2.9593505859375}, {"k": 3, "reads_keep1": 2.976318359375, "reads_keep3": 2.9468994140625}], "step_s": 0.8192, "grad_norm": 0.2002, "grad_norm_mean": 0.1975, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
81
+ {"TRUNK_CHECKSUM": 6000, "sum": -23256.11460387953, "abs": 5922086.545251617}
82
+ {"step": 6000, "loss": 2.691514492034912, "lr": 0.0001231928003835568, "elapsed_s": 4576.404948949814, "util": [0.9858, 0.9985], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.991455078125, "reads_keep3": 2.960693359375}, {"k": 3, "reads_keep1": 2.9720458984375, "reads_keep3": 2.9393310546875}], "step_s": 0.7953, "grad_norm": 0.20242, "grad_norm_mean": 0.2039, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
83
+ {"step": 6100, "loss": 2.883620262145996, "lr": 0.00011594440529630196, "elapsed_s": 4657.44682765007, "util": [0.9902, 0.9985], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9918212890625, "reads_keep3": 2.9710693359375}, {"k": 3, "reads_keep1": 2.976318359375, "reads_keep3": 2.947509765625}], "step_s": 0.8104, "grad_norm": 0.21226, "grad_norm_mean": 0.19976, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
84
+ {"step": 6200, "loss": 2.6363463401794434, "lr": 0.00010908926170699825, "elapsed_s": 4737.542729854584, "util": [0.9873, 0.999], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9923095703125, "reads_keep3": 2.96728515625}, {"k": 3, "reads_keep1": 2.972900390625, "reads_keep3": 2.946533203125}], "step_s": 0.801, "grad_norm": 0.18963, "grad_norm_mean": 0.2015, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
85
+ {"step": 6300, "loss": 2.6801962852478027, "lr": 0.00010263996351906082, "elapsed_s": 4817.433548927307, "util": [0.9878, 1.0], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9910888671875, "reads_keep3": 2.963134765625}, {"k": 3, "reads_keep1": 2.97265625, "reads_keep3": 2.945068359375}], "step_s": 0.7989, "grad_norm": 0.19201, "grad_norm_mean": 0.19929, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
86
+ {"step": 6400, "loss": 2.8494482040405273, "lr": 9.660835903842457e-05, "elapsed_s": 4899.8473970890045, "util": [0.9897, 0.9995], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9908447265625, "reads_keep3": 2.95361328125}, {"k": 3, "reads_keep1": 2.9683837890625, "reads_keep3": 2.929931640625}], "step_s": 0.8242, "grad_norm": 0.20683, "grad_norm_mean": 0.19865, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
87
+ {"step": 6500, "loss": 2.900207996368408, "lr": 9.100552920646992e-05, "elapsed_s": 4980.584800720215, "util": [0.9907, 0.9985], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.993408203125, "reads_keep3": 2.9638671875}, {"k": 3, "reads_keep1": 2.9718017578125, "reads_keep3": 2.943115234375}], "step_s": 0.8074, "grad_norm": 0.19677, "grad_norm_mean": 0.19913, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
88
+ {"step": 6600, "loss": 2.6298065185546875, "lr": 8.584176724270953e-05, "elapsed_s": 5061.568926811218, "util": [0.9941, 0.999], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.989013671875, "reads_keep3": 2.95654296875}, {"k": 3, "reads_keep1": 2.9732666015625, "reads_keep3": 2.94384765625}], "step_s": 0.8098, "grad_norm": 0.19845, "grad_norm_mean": 0.19698, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
89
+ {"step": 6700, "loss": 2.5884947776794434, "lr": 8.112655973463532e-05, "elapsed_s": 5140.840366840363, "util": [0.9907, 1.0], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.98779296875, "reads_keep3": 2.9552001953125}, {"k": 3, "reads_keep1": 2.9674072265625, "reads_keep3": 2.9361572265625}], "step_s": 0.7927, "grad_norm": 0.19382, "grad_norm_mean": 0.19712, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
90
+ {"step": 6800, "loss": 2.826439380645752, "lr": 7.686856920946722e-05, "elapsed_s": 5222.547169446945, "util": [0.9932, 0.999], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9931640625, "reads_keep3": 2.9654541015625}, {"k": 3, "reads_keep1": 2.9737548828125, "reads_keep3": 2.9482421875}], "step_s": 0.817, "grad_norm": 0.20056, "grad_norm_mean": 0.19746, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
91
+ {"step": 6900, "loss": 2.579041004180908, "lr": 7.307561821982112e-05, "elapsed_s": 5312.427621841431, "util": [0.9907, 0.9995], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9896240234375, "reads_keep3": 2.9691162109375}, {"k": 3, "reads_keep1": 2.971435546875, "reads_keep3": 2.9505615234375}], "step_s": 0.8988, "grad_norm": 0.19895, "grad_norm_mean": 0.19855, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
92
+ {"TRUNK_CHECKSUM": 7000, "sum": -24148.91697897852, "abs": 5896411.156199637}
93
+ {"step": 7000, "loss": 2.8963675498962402, "lr": 6.975467497253317e-05, "elapsed_s": 5350.748041152954, "util": [0.9937, 0.9985], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9918212890625, "reads_keep3": 2.967529296875}, {"k": 3, "reads_keep1": 2.970458984375, "reads_keep3": 2.946533203125}], "step_s": 0.3835, "grad_norm": 0.20476, "grad_norm_mean": 0.19756, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
94
+ {"step": 7100, "loss": 2.4268391132354736, "lr": 6.691184052704267e-05, "elapsed_s": 5378.028213739395, "util": [0.9922, 0.9976], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9952392578125, "reads_keep3": 2.9696044921875}, {"k": 3, "reads_keep1": 2.9754638671875, "reads_keep3": 2.939453125}], "step_s": 0.2728, "grad_norm": 0.19741, "grad_norm_mean": 0.19865, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
95
+ {"step": 7200, "loss": 3.0772056579589844, "lr": 6.455233758685185e-05, "elapsed_s": 5405.304255485535, "util": [0.9912, 0.9985], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9912109375, "reads_keep3": 2.9609375}, {"k": 3, "reads_keep1": 2.974365234375, "reads_keep3": 2.9376220703125}], "step_s": 0.2728, "grad_norm": 0.20115, "grad_norm_mean": 0.19978, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
96
+ {"step": 7300, "loss": 2.7156317234039307, "lr": 6.268050090465387e-05, "elapsed_s": 5432.576008319855, "util": [0.9946, 1.0], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.990478515625, "reads_keep3": 2.9658203125}, {"k": 3, "reads_keep1": 2.9688720703125, "reads_keep3": 2.941162109375}], "step_s": 0.2727, "grad_norm": 0.20153, "grad_norm_mean": 0.19968, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
97
+ {"step": 7400, "loss": 2.7013535499572754, "lr": 6.12997693187572e-05, "elapsed_s": 5459.862614631653, "util": [0.9956, 0.9995], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.9925537109375, "reads_keep3": 2.9715576171875}, {"k": 3, "reads_keep1": 2.97021484375, "reads_keep3": 2.9449462890625}], "step_s": 0.2729, "grad_norm": 0.20484, "grad_norm_mean": 0.20115, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
98
+ {"step": 7500, "loss": 2.566859245300293, "lr": 6.0412679435435414e-05, "elapsed_s": 5487.134707212448, "util": [0.9941, 0.9985], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.99365234375, "reads_keep3": 2.9697265625}, {"k": 3, "reads_keep1": 2.9718017578125, "reads_keep3": 2.9490966796875}], "step_s": 0.2727, "grad_norm": 0.19749, "grad_norm_mean": 0.20082, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
99
+ {"step": 7600, "loss": 2.9380922317504883, "lr": 6.0020860968809794e-05, "elapsed_s": 5514.400843381882, "util": [0.9932, 0.9985], "drop": [0.0, 0.0], "traffic": [{"k": 3, "reads_keep1": 2.993408203125, "reads_keep3": 2.961669921875}, {"k": 3, "reads_keep1": 2.97216796875, "reads_keep3": 2.936767578125}], "step_s": 0.2727, "grad_norm": 0.19654, "grad_norm_mean": 0.20245, "clip_frac": 0.0, "pairs_per_step": 786432.0, "max_mem_alloc_gb": [21.7, 21.72, 21.71, 21.71, 21.7, 21.7, 21.7, 21.72], "max_mem_reserved_gb": [22.04, 22.06, 22.06, 22.06, 22.06, 22.04, 22.05, 22.09]}
100
+ {"DONE": "tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn", "main_on": 2.839003264904022, "main_off": 4.719659045338631}
101
+ END rc=0 Sat Oct 24 03:13:45 EDT 2026
in-progress/t3-archive/orchard-flame-15/gate_b_tier/result_tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn.json ADDED
@@ -0,0 +1,4044 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "arm": "tk-b220-e512-1e9-g4-lrn-sh1-cf0-sn",
3
+ "base": "tk-b220-e512",
4
+ "mult": 1,
5
+ "init_seed": 0,
6
+ "steps": 7629,
7
+ "world": 8,
8
+ "train_seqs": 488281,
9
+ "non_embed": 3240825984,
10
+ "added_params": 3021371136,
11
+ "final_eval_on": {
12
+ "main_heldout": {
13
+ "loss": 2.839003264904022,
14
+ "ppl": 17.098714152209062,
15
+ "tokens": 524288,
16
+ "kind": "in_domain"
17
+ },
18
+ "xd_lang": {
19
+ "loss": 5.746570184826851,
20
+ "ppl": 313.11489029248736,
21
+ "tokens": 524288,
22
+ "kind": "cross_domain"
23
+ },
24
+ "xd_code": {
25
+ "loss": 3.5750409811735153,
26
+ "ppl": 35.69608365307928,
27
+ "tokens": 524288,
28
+ "kind": "cross_domain"
29
+ }
30
+ },
31
+ "final_eval_off": {
32
+ "main_heldout": {
33
+ "loss": 4.719659045338631,
34
+ "ppl": 112.13001489824778,
35
+ "tokens": 524288,
36
+ "kind": "in_domain"
37
+ },
38
+ "xd_lang": {
39
+ "loss": 6.385182589292526,
40
+ "ppl": 592.9929968861693,
41
+ "tokens": 524288,
42
+ "kind": "cross_domain"
43
+ },
44
+ "xd_code": {
45
+ "loss": 4.02534493803978,
46
+ "ppl": 55.99962185033607,
47
+ "tokens": 524288,
48
+ "kind": "cross_domain"
49
+ }
50
+ },
51
+ "traffic": {
52
+ "3": {
53
+ "k": 3,
54
+ "reads_keep1": 2.94268798828125,
55
+ "reads_keep3": 2.81689453125
56
+ },
57
+ "9": {
58
+ "k": 3,
59
+ "reads_keep1": 2.94677734375,
60
+ "reads_keep3": 2.857421875
61
+ }
62
+ },
63
+ "history": [
64
+ {
65
+ "step": 0,
66
+ "loss": 10.603790283203125,
67
+ "lr": 2e-06,
68
+ "elapsed_s": 14.28202748298645,
69
+ "util": [
70
+ 0.7944,
71
+ 0.5996
72
+ ],
73
+ "drop": [
74
+ 0.0,
75
+ 0.0
76
+ ],
77
+ "traffic": [
78
+ {
79
+ "k": 3,
80
+ "reads_keep1": 2.9078369140625,
81
+ "reads_keep3": 2.7408447265625
82
+ },
83
+ {
84
+ "k": 3,
85
+ "reads_keep1": 2.788818359375,
86
+ "reads_keep3": 2.468994140625
87
+ }
88
+ ],
89
+ "step_s": 14.2272,
90
+ "grad_norm": 11.13349,
91
+ "grad_norm_mean": 11.13349,
92
+ "clip_frac": 1.0,
93
+ "pairs_per_step": 786432.0,
94
+ "max_mem_alloc_gb": [
95
+ 16.87,
96
+ 16.88,
97
+ 16.87,
98
+ 16.88,
99
+ 16.89,
100
+ 16.91,
101
+ 16.9,
102
+ 16.89
103
+ ],
104
+ "max_mem_reserved_gb": [
105
+ 17.22,
106
+ 17.24,
107
+ 17.2,
108
+ 17.22,
109
+ 17.26,
110
+ 17.26,
111
+ 17.26,
112
+ 17.26
113
+ ]
114
+ },
115
+ {
116
+ "step": 100,
117
+ "loss": 5.4917120933532715,
118
+ "lr": 0.00020199999999999998,
119
+ "elapsed_s": 91.6028208732605,
120
+ "util": [
121
+ 0.9331,
122
+ 0.8555
123
+ ],
124
+ "drop": [
125
+ 0.0,
126
+ 0.0
127
+ ],
128
+ "traffic": [
129
+ {
130
+ "k": 3,
131
+ "reads_keep1": 2.979248046875,
132
+ "reads_keep3": 2.91796875
133
+ },
134
+ {
135
+ "k": 3,
136
+ "reads_keep1": 2.9642333984375,
137
+ "reads_keep3": 2.8978271484375
138
+ }
139
+ ],
140
+ "step_s": 0.7733,
141
+ "grad_norm": 0.90461,
142
+ "grad_norm_mean": 2.08934,
143
+ "clip_frac": 0.83,
144
+ "pairs_per_step": 786432.0,
145
+ "max_mem_alloc_gb": [
146
+ 21.7,
147
+ 21.72,
148
+ 21.71,
149
+ 21.71,
150
+ 21.69,
151
+ 21.68,
152
+ 21.68,
153
+ 21.71
154
+ ],
155
+ "max_mem_reserved_gb": [
156
+ 22.04,
157
+ 22.06,
158
+ 22.06,
159
+ 22.06,
160
+ 22.04,
161
+ 22.02,
162
+ 22.04,
163
+ 22.07
164
+ ]
165
+ },
166
+ {
167
+ "step": 200,
168
+ "loss": 4.816793918609619,
169
+ "lr": 0.000402,
170
+ "elapsed_s": 167.11967039108276,
171
+ "util": [
172
+ 0.9146,
173
+ 0.8867
174
+ ],
175
+ "drop": [
176
+ 0.0,
177
+ 0.0
178
+ ],
179
+ "traffic": [
180
+ {
181
+ "k": 3,
182
+ "reads_keep1": 2.9893798828125,
183
+ "reads_keep3": 2.941650390625
184
+ },
185
+ {
186
+ "k": 3,
187
+ "reads_keep1": 2.9835205078125,
188
+ "reads_keep3": 2.941162109375
189
+ }
190
+ ],
191
+ "step_s": 0.7553,
192
+ "grad_norm": 0.66178,
193
+ "grad_norm_mean": 0.89119,
194
+ "clip_frac": 0.19,
195
+ "pairs_per_step": 786432.0,
196
+ "max_mem_alloc_gb": [
197
+ 21.7,
198
+ 21.72,
199
+ 21.71,
200
+ 21.71,
201
+ 21.7,
202
+ 21.68,
203
+ 21.7,
204
+ 21.72
205
+ ],
206
+ "max_mem_reserved_gb": [
207
+ 22.04,
208
+ 22.06,
209
+ 22.06,
210
+ 22.06,
211
+ 22.06,
212
+ 22.02,
213
+ 22.04,
214
+ 22.09
215
+ ]
216
+ },
217
+ {
218
+ "step": 300,
219
+ "loss": 4.762606620788574,
220
+ "lr": 0.0006,
221
+ "elapsed_s": 242.11455154418945,
222
+ "util": [
223
+ 0.877,
224
+ 0.8628
225
+ ],
226
+ "drop": [
227
+ 0.0,
228
+ 0.0
229
+ ],
230
+ "traffic": [
231
+ {
232
+ "k": 3,
233
+ "reads_keep1": 2.970458984375,
234
+ "reads_keep3": 2.91796875
235
+ },
236
+ {
237
+ "k": 3,
238
+ "reads_keep1": 2.9696044921875,
239
+ "reads_keep3": 2.913818359375
240
+ }
241
+ ],
242
+ "step_s": 0.75,
243
+ "grad_norm": 1.28636,
244
+ "grad_norm_mean": 2.65395,
245
+ "clip_frac": 0.49,
246
+ "pairs_per_step": 786432.0,
247
+ "max_mem_alloc_gb": [
248
+ 21.7,
249
+ 21.72,
250
+ 21.71,
251
+ 21.71,
252
+ 21.7,
253
+ 21.7,
254
+ 21.7,
255
+ 21.72
256
+ ],
257
+ "max_mem_reserved_gb": [
258
+ 22.04,
259
+ 22.06,
260
+ 22.06,
261
+ 22.06,
262
+ 22.06,
263
+ 22.04,
264
+ 22.04,
265
+ 22.09
266
+ ]
267
+ },
268
+ {
269
+ "step": 400,
270
+ "loss": 4.119720458984375,
271
+ "lr": 0.0005997519852153506,
272
+ "elapsed_s": 317.84852719306946,
273
+ "util": [
274
+ 0.9248,
275
+ 0.9292
276
+ ],
277
+ "drop": [
278
+ 0.0,
279
+ 0.0
280
+ ],
281
+ "traffic": [
282
+ {
283
+ "k": 3,
284
+ "reads_keep1": 2.9630126953125,
285
+ "reads_keep3": 2.9049072265625
286
+ },
287
+ {
288
+ "k": 3,
289
+ "reads_keep1": 2.957763671875,
290
+ "reads_keep3": 2.9110107421875
291
+ }
292
+ ],
293
+ "step_s": 0.7573,
294
+ "grad_norm": 0.42827,
295
+ "grad_norm_mean": 0.64226,
296
+ "clip_frac": 0.08,
297
+ "pairs_per_step": 786432.0,
298
+ "max_mem_alloc_gb": [
299
+ 21.7,
300
+ 21.72,
301
+ 21.71,
302
+ 21.71,
303
+ 21.7,
304
+ 21.7,
305
+ 21.7,
306
+ 21.72
307
+ ],
308
+ "max_mem_reserved_gb": [
309
+ 22.04,
310
+ 22.06,
311
+ 22.06,
312
+ 22.06,
313
+ 22.06,
314
+ 22.04,
315
+ 22.04,
316
+ 22.09
317
+ ]
318
+ },
319
+ {
320
+ "step": 500,
321
+ "loss": 3.9423646926879883,
322
+ "lr": 0.0005990083965009096,
323
+ "elapsed_s": 392.6189374923706,
324
+ "util": [
325
+ 0.9316,
326
+ 0.9556
327
+ ],
328
+ "drop": [
329
+ 0.0,
330
+ 0.0
331
+ ],
332
+ "traffic": [
333
+ {
334
+ "k": 3,
335
+ "reads_keep1": 2.975830078125,
336
+ "reads_keep3": 2.9346923828125
337
+ },
338
+ {
339
+ "k": 3,
340
+ "reads_keep1": 2.9722900390625,
341
+ "reads_keep3": 2.938720703125
342
+ }
343
+ ],
344
+ "step_s": 0.7477,
345
+ "grad_norm": 0.32777,
346
+ "grad_norm_mean": 0.64873,
347
+ "clip_frac": 0.13,
348
+ "pairs_per_step": 786432.0,
349
+ "max_mem_alloc_gb": [
350
+ 21.7,
351
+ 21.72,
352
+ 21.71,
353
+ 21.71,
354
+ 21.7,
355
+ 21.7,
356
+ 21.7,
357
+ 21.72
358
+ ],
359
+ "max_mem_reserved_gb": [
360
+ 22.04,
361
+ 22.06,
362
+ 22.06,
363
+ 22.06,
364
+ 22.06,
365
+ 22.04,
366
+ 22.05,
367
+ 22.09
368
+ ]
369
+ },
370
+ {
371
+ "step": 600,
372
+ "loss": 3.8606295585632324,
373
+ "lr": 0.0005977705999381204,
374
+ "elapsed_s": 467.8239600658417,
375
+ "util": [
376
+ 0.8862,
377
+ 0.9526
378
+ ],
379
+ "drop": [
380
+ 0.0,
381
+ 0.0
382
+ ],
383
+ "traffic": [
384
+ {
385
+ "k": 3,
386
+ "reads_keep1": 2.98681640625,
387
+ "reads_keep3": 2.9281005859375
388
+ },
389
+ {
390
+ "k": 3,
391
+ "reads_keep1": 2.9840087890625,
392
+ "reads_keep3": 2.9398193359375
393
+ }
394
+ ],
395
+ "step_s": 0.752,
396
+ "grad_norm": 1.06046,
397
+ "grad_norm_mean": 0.49575,
398
+ "clip_frac": 0.07,
399
+ "pairs_per_step": 786432.0,
400
+ "max_mem_alloc_gb": [
401
+ 21.7,
402
+ 21.72,
403
+ 21.71,
404
+ 21.71,
405
+ 21.7,
406
+ 21.7,
407
+ 21.7,
408
+ 21.72
409
+ ],
410
+ "max_mem_reserved_gb": [
411
+ 22.04,
412
+ 22.06,
413
+ 22.06,
414
+ 22.06,
415
+ 22.06,
416
+ 22.04,
417
+ 22.05,
418
+ 22.09
419
+ ]
420
+ },
421
+ {
422
+ "step": 700,
423
+ "loss": 3.6110830307006836,
424
+ "lr": 0.0005960408695406716,
425
+ "elapsed_s": 541.3506631851196,
426
+ "util": [
427
+ 0.8955,
428
+ 0.9541
429
+ ],
430
+ "drop": [
431
+ 0.0,
432
+ 0.0
433
+ ],
434
+ "traffic": [
435
+ {
436
+ "k": 3,
437
+ "reads_keep1": 2.9793701171875,
438
+ "reads_keep3": 2.9197998046875
439
+ },
440
+ {
441
+ "k": 3,
442
+ "reads_keep1": 2.97119140625,
443
+ "reads_keep3": 2.92822265625
444
+ }
445
+ ],
446
+ "step_s": 0.7353,
447
+ "grad_norm": 2.27035,
448
+ "grad_norm_mean": 0.59618,
449
+ "clip_frac": 0.13,
450
+ "pairs_per_step": 786432.0,
451
+ "max_mem_alloc_gb": [
452
+ 21.7,
453
+ 21.72,
454
+ 21.71,
455
+ 21.71,
456
+ 21.7,
457
+ 21.7,
458
+ 21.7,
459
+ 21.72
460
+ ],
461
+ "max_mem_reserved_gb": [
462
+ 22.04,
463
+ 22.06,
464
+ 22.06,
465
+ 22.06,
466
+ 22.06,
467
+ 22.04,
468
+ 22.05,
469
+ 22.09
470
+ ]
471
+ },
472
+ {
473
+ "step": 800,
474
+ "loss": 3.4680235385894775,
475
+ "lr": 0.0005938223830768002,
476
+ "elapsed_s": 615.0662519931793,
477
+ "util": [
478
+ 0.8975,
479
+ 0.9688
480
+ ],
481
+ "drop": [
482
+ 0.0,
483
+ 0.0
484
+ ],
485
+ "traffic": [
486
+ {
487
+ "k": 3,
488
+ "reads_keep1": 2.9862060546875,
489
+ "reads_keep3": 2.9453125
490
+ },
491
+ {
492
+ "k": 3,
493
+ "reads_keep1": 2.978515625,
494
+ "reads_keep3": 2.9522705078125
495
+ }
496
+ ],
497
+ "step_s": 0.7371,
498
+ "grad_norm": 0.69486,
499
+ "grad_norm_mean": 0.42549,
500
+ "clip_frac": 0.05,
501
+ "pairs_per_step": 786432.0,
502
+ "max_mem_alloc_gb": [
503
+ 21.7,
504
+ 21.72,
505
+ 21.71,
506
+ 21.71,
507
+ 21.7,
508
+ 21.7,
509
+ 21.7,
510
+ 21.72
511
+ ],
512
+ "max_mem_reserved_gb": [
513
+ 22.04,
514
+ 22.06,
515
+ 22.06,
516
+ 22.06,
517
+ 22.06,
518
+ 22.04,
519
+ 22.05,
520
+ 22.09
521
+ ]
522
+ },
523
+ {
524
+ "step": 900,
525
+ "loss": 3.5275583267211914,
526
+ "lr": 0.0005911192162312665,
527
+ "elapsed_s": 687.6100196838379,
528
+ "util": [
529
+ 0.917,
530
+ 0.9854
531
+ ],
532
+ "drop": [
533
+ 0.0,
534
+ 0.0
535
+ ],
536
+ "traffic": [
537
+ {
538
+ "k": 3,
539
+ "reads_keep1": 2.9888916015625,
540
+ "reads_keep3": 2.953369140625
541
+ },
542
+ {
543
+ "k": 3,
544
+ "reads_keep1": 2.9837646484375,
545
+ "reads_keep3": 2.9569091796875
546
+ }
547
+ ],
548
+ "step_s": 0.7255,
549
+ "grad_norm": 0.25159,
550
+ "grad_norm_mean": 0.28878,
551
+ "clip_frac": 0.0,
552
+ "pairs_per_step": 786432.0,
553
+ "max_mem_alloc_gb": [
554
+ 21.7,
555
+ 21.72,
556
+ 21.71,
557
+ 21.71,
558
+ 21.7,
559
+ 21.7,
560
+ 21.7,
561
+ 21.72
562
+ ],
563
+ "max_mem_reserved_gb": [
564
+ 22.04,
565
+ 22.06,
566
+ 22.06,
567
+ 22.06,
568
+ 22.06,
569
+ 22.04,
570
+ 22.05,
571
+ 22.09
572
+ ]
573
+ },
574
+ {
575
+ "step": 1000,
576
+ "loss": 3.5475330352783203,
577
+ "lr": 0.0005879363351177225,
578
+ "elapsed_s": 761.1710834503174,
579
+ "util": [
580
+ 0.9072,
581
+ 0.9805
582
+ ],
583
+ "drop": [
584
+ 0.0,
585
+ 0.0
586
+ ],
587
+ "traffic": [
588
+ {
589
+ "k": 3,
590
+ "reads_keep1": 2.980224609375,
591
+ "reads_keep3": 2.9296875
592
+ },
593
+ {
594
+ "k": 3,
595
+ "reads_keep1": 2.9739990234375,
596
+ "reads_keep3": 2.939208984375
597
+ }
598
+ ],
599
+ "step_s": 0.7355,
600
+ "grad_norm": 0.25038,
601
+ "grad_norm_mean": 0.30107,
602
+ "clip_frac": 0.02,
603
+ "pairs_per_step": 786432.0,
604
+ "max_mem_alloc_gb": [
605
+ 21.7,
606
+ 21.72,
607
+ 21.71,
608
+ 21.71,
609
+ 21.7,
610
+ 21.7,
611
+ 21.7,
612
+ 21.72
613
+ ],
614
+ "max_mem_reserved_gb": [
615
+ 22.04,
616
+ 22.06,
617
+ 22.06,
618
+ 22.06,
619
+ 22.06,
620
+ 22.04,
621
+ 22.05,
622
+ 22.09
623
+ ]
624
+ },
625
+ {
626
+ "step": 1100,
627
+ "loss": 3.6279101371765137,
628
+ "lr": 0.0005842795871552345,
629
+ "elapsed_s": 834.9443986415863,
630
+ "util": [
631
+ 0.9219,
632
+ 0.9858
633
+ ],
634
+ "drop": [
635
+ 0.0,
636
+ 0.0
637
+ ],
638
+ "traffic": [
639
+ {
640
+ "k": 3,
641
+ "reads_keep1": 2.9876708984375,
642
+ "reads_keep3": 2.950439453125
643
+ },
644
+ {
645
+ "k": 3,
646
+ "reads_keep1": 2.98291015625,
647
+ "reads_keep3": 2.954833984375
648
+ }
649
+ ],
650
+ "step_s": 0.7378,
651
+ "grad_norm": 0.24847,
652
+ "grad_norm_mean": 0.25598,
653
+ "clip_frac": 0.0,
654
+ "pairs_per_step": 786432.0,
655
+ "max_mem_alloc_gb": [
656
+ 21.7,
657
+ 21.72,
658
+ 21.71,
659
+ 21.71,
660
+ 21.7,
661
+ 21.7,
662
+ 21.7,
663
+ 21.72
664
+ ],
665
+ "max_mem_reserved_gb": [
666
+ 22.04,
667
+ 22.06,
668
+ 22.06,
669
+ 22.06,
670
+ 22.06,
671
+ 22.04,
672
+ 22.05,
673
+ 22.09
674
+ ]
675
+ },
676
+ {
677
+ "step": 1200,
678
+ "loss": 3.535391092300415,
679
+ "lr": 0.0005801556903257169,
680
+ "elapsed_s": 909.2344558238983,
681
+ "util": [
682
+ 0.9189,
683
+ 0.981
684
+ ],
685
+ "drop": [
686
+ 0.0,
687
+ 0.0
688
+ ],
689
+ "traffic": [
690
+ {
691
+ "k": 3,
692
+ "reads_keep1": 2.9906005859375,
693
+ "reads_keep3": 2.956787109375
694
+ },
695
+ {
696
+ "k": 3,
697
+ "reads_keep1": 2.98095703125,
698
+ "reads_keep3": 2.9573974609375
699
+ }
700
+ ],
701
+ "step_s": 0.7429,
702
+ "grad_norm": 0.23094,
703
+ "grad_norm_mean": 0.24032,
704
+ "clip_frac": 0.0,
705
+ "pairs_per_step": 786432.0,
706
+ "max_mem_alloc_gb": [
707
+ 21.7,
708
+ 21.72,
709
+ 21.71,
710
+ 21.71,
711
+ 21.7,
712
+ 21.7,
713
+ 21.7,
714
+ 21.72
715
+ ],
716
+ "max_mem_reserved_gb": [
717
+ 22.04,
718
+ 22.06,
719
+ 22.06,
720
+ 22.06,
721
+ 22.06,
722
+ 22.04,
723
+ 22.05,
724
+ 22.09
725
+ ]
726
+ },
727
+ {
728
+ "step": 1300,
729
+ "loss": 3.3749685287475586,
730
+ "lr": 0.0005755722208320148,
731
+ "elapsed_s": 984.4957759380341,
732
+ "util": [
733
+ 0.9287,
734
+ 0.9917
735
+ ],
736
+ "drop": [
737
+ 0.0,
738
+ 0.0
739
+ ],
740
+ "traffic": [
741
+ {
742
+ "k": 3,
743
+ "reads_keep1": 2.9925537109375,
744
+ "reads_keep3": 2.95458984375
745
+ },
746
+ {
747
+ "k": 3,
748
+ "reads_keep1": 2.98291015625,
749
+ "reads_keep3": 2.9512939453125
750
+ }
751
+ ],
752
+ "step_s": 0.7526,
753
+ "grad_norm": 0.2391,
754
+ "grad_norm_mean": 0.25197,
755
+ "clip_frac": 0.01,
756
+ "pairs_per_step": 786432.0,
757
+ "max_mem_alloc_gb": [
758
+ 21.7,
759
+ 21.72,
760
+ 21.71,
761
+ 21.71,
762
+ 21.7,
763
+ 21.7,
764
+ 21.7,
765
+ 21.72
766
+ ],
767
+ "max_mem_reserved_gb": [
768
+ 22.04,
769
+ 22.06,
770
+ 22.06,
771
+ 22.06,
772
+ 22.06,
773
+ 22.04,
774
+ 22.05,
775
+ 22.09
776
+ ]
777
+ },
778
+ {
779
+ "step": 1400,
780
+ "loss": 3.142401695251465,
781
+ "lr": 0.000570537599179309,
782
+ "elapsed_s": 1059.1784732341766,
783
+ "util": [
784
+ 0.9165,
785
+ 0.9897
786
+ ],
787
+ "drop": [
788
+ 0.0,
789
+ 0.0
790
+ ],
791
+ "traffic": [
792
+ {
793
+ "k": 3,
794
+ "reads_keep1": 2.981689453125,
795
+ "reads_keep3": 2.920166015625
796
+ },
797
+ {
798
+ "k": 3,
799
+ "reads_keep1": 2.9710693359375,
800
+ "reads_keep3": 2.91748046875
801
+ }
802
+ ],
803
+ "step_s": 0.7468,
804
+ "grad_norm": 0.2181,
805
+ "grad_norm_mean": 0.22604,
806
+ "clip_frac": 0.0,
807
+ "pairs_per_step": 786432.0,
808
+ "max_mem_alloc_gb": [
809
+ 21.7,
810
+ 21.72,
811
+ 21.71,
812
+ 21.71,
813
+ 21.7,
814
+ 21.7,
815
+ 21.7,
816
+ 21.72
817
+ ],
818
+ "max_mem_reserved_gb": [
819
+ 22.04,
820
+ 22.06,
821
+ 22.06,
822
+ 22.06,
823
+ 22.06,
824
+ 22.04,
825
+ 22.05,
826
+ 22.09
827
+ ]
828
+ },
829
+ {
830
+ "step": 1500,
831
+ "loss": 3.2960891723632812,
832
+ "lr": 0.0005650610747054142,
833
+ "elapsed_s": 1134.9925062656403,
834
+ "util": [
835
+ 0.9243,
836
+ 0.9893
837
+ ],
838
+ "drop": [
839
+ 0.0,
840
+ 0.0
841
+ ],
842
+ "traffic": [
843
+ {
844
+ "k": 3,
845
+ "reads_keep1": 2.991943359375,
846
+ "reads_keep3": 2.940185546875
847
+ },
848
+ {
849
+ "k": 3,
850
+ "reads_keep1": 2.982177734375,
851
+ "reads_keep3": 2.9522705078125
852
+ }
853
+ ],
854
+ "step_s": 0.7582,
855
+ "grad_norm": 0.20629,
856
+ "grad_norm_mean": 0.22417,
857
+ "clip_frac": 0.0,
858
+ "pairs_per_step": 786432.0,
859
+ "max_mem_alloc_gb": [
860
+ 21.7,
861
+ 21.72,
862
+ 21.71,
863
+ 21.71,
864
+ 21.7,
865
+ 21.7,
866
+ 21.7,
867
+ 21.72
868
+ ],
869
+ "max_mem_reserved_gb": [
870
+ 22.04,
871
+ 22.06,
872
+ 22.06,
873
+ 22.06,
874
+ 22.06,
875
+ 22.04,
876
+ 22.05,
877
+ 22.09
878
+ ]
879
+ },
880
+ {
881
+ "step": 1600,
882
+ "loss": 3.2592875957489014,
883
+ "lr": 0.0005591527085883902,
884
+ "elapsed_s": 1209.1713495254517,
885
+ "util": [
886
+ 0.9263,
887
+ 0.9888
888
+ ],
889
+ "drop": [
890
+ 0.0,
891
+ 0.0
892
+ ],
893
+ "traffic": [
894
+ {
895
+ "k": 3,
896
+ "reads_keep1": 2.9864501953125,
897
+ "reads_keep3": 2.9544677734375
898
+ },
899
+ {
900
+ "k": 3,
901
+ "reads_keep1": 2.9810791015625,
902
+ "reads_keep3": 2.9556884765625
903
+ }
904
+ ],
905
+ "step_s": 0.7417,
906
+ "grad_norm": 0.23391,
907
+ "grad_norm_mean": 0.24642,
908
+ "clip_frac": 0.01,
909
+ "pairs_per_step": 786432.0,
910
+ "max_mem_alloc_gb": [
911
+ 21.7,
912
+ 21.72,
913
+ 21.71,
914
+ 21.71,
915
+ 21.7,
916
+ 21.7,
917
+ 21.7,
918
+ 21.72
919
+ ],
920
+ "max_mem_reserved_gb": [
921
+ 22.04,
922
+ 22.06,
923
+ 22.06,
924
+ 22.06,
925
+ 22.06,
926
+ 22.04,
927
+ 22.05,
928
+ 22.09
929
+ ]
930
+ },
931
+ {
932
+ "step": 1700,
933
+ "loss": 3.318453311920166,
934
+ "lr": 0.000552823355362682,
935
+ "elapsed_s": 1282.6599502563477,
936
+ "util": [
937
+ 0.9297,
938
+ 0.9917
939
+ ],
940
+ "drop": [
941
+ 0.0,
942
+ 0.0
943
+ ],
944
+ "traffic": [
945
+ {
946
+ "k": 3,
947
+ "reads_keep1": 2.9873046875,
948
+ "reads_keep3": 2.9434814453125
949
+ },
950
+ {
951
+ "k": 3,
952
+ "reads_keep1": 2.972900390625,
953
+ "reads_keep3": 2.9423828125
954
+ }
955
+ ],
956
+ "step_s": 0.7349,
957
+ "grad_norm": 0.20709,
958
+ "grad_norm_mean": 0.2237,
959
+ "clip_frac": 0.0,
960
+ "pairs_per_step": 786432.0,
961
+ "max_mem_alloc_gb": [
962
+ 21.7,
963
+ 21.72,
964
+ 21.71,
965
+ 21.71,
966
+ 21.7,
967
+ 21.7,
968
+ 21.7,
969
+ 21.72
970
+ ],
971
+ "max_mem_reserved_gb": [
972
+ 22.04,
973
+ 22.06,
974
+ 22.06,
975
+ 22.06,
976
+ 22.06,
977
+ 22.04,
978
+ 22.05,
979
+ 22.09
980
+ ]
981
+ },
982
+ {
983
+ "step": 1800,
984
+ "loss": 3.2907724380493164,
985
+ "lr": 0.0005460846429777509,
986
+ "elapsed_s": 1358.0658473968506,
987
+ "util": [
988
+ 0.9307,
989
+ 0.9897
990
+ ],
991
+ "drop": [
992
+ 0.0,
993
+ 0.0
994
+ ],
995
+ "traffic": [
996
+ {
997
+ "k": 3,
998
+ "reads_keep1": 2.9906005859375,
999
+ "reads_keep3": 2.96044921875
1000
+ },
1001
+ {
1002
+ "k": 3,
1003
+ "reads_keep1": 2.981689453125,
1004
+ "reads_keep3": 2.95849609375
1005
+ }
1006
+ ],
1007
+ "step_s": 0.7542,
1008
+ "grad_norm": 0.21604,
1009
+ "grad_norm_mean": 0.23864,
1010
+ "clip_frac": 0.01,
1011
+ "pairs_per_step": 786432.0,
1012
+ "max_mem_alloc_gb": [
1013
+ 21.7,
1014
+ 21.72,
1015
+ 21.71,
1016
+ 21.71,
1017
+ 21.7,
1018
+ 21.7,
1019
+ 21.7,
1020
+ 21.72
1021
+ ],
1022
+ "max_mem_reserved_gb": [
1023
+ 22.04,
1024
+ 22.06,
1025
+ 22.06,
1026
+ 22.06,
1027
+ 22.06,
1028
+ 22.04,
1029
+ 22.05,
1030
+ 22.09
1031
+ ]
1032
+ },
1033
+ {
1034
+ "step": 1900,
1035
+ "loss": 3.3151278495788574,
1036
+ "lr": 0.0005389489514358262,
1037
+ "elapsed_s": 1431.3323302268982,
1038
+ "util": [
1039
+ 0.9346,
1040
+ 0.9927
1041
+ ],
1042
+ "drop": [
1043
+ 0.0,
1044
+ 0.0
1045
+ ],
1046
+ "traffic": [
1047
+ {
1048
+ "k": 3,
1049
+ "reads_keep1": 2.9892578125,
1050
+ "reads_keep3": 2.9520263671875
1051
+ },
1052
+ {
1053
+ "k": 3,
1054
+ "reads_keep1": 2.9786376953125,
1055
+ "reads_keep3": 2.9476318359375
1056
+ }
1057
+ ],
1058
+ "step_s": 0.7325,
1059
+ "grad_norm": 0.20736,
1060
+ "grad_norm_mean": 0.21179,
1061
+ "clip_frac": 0.0,
1062
+ "pairs_per_step": 786432.0,
1063
+ "max_mem_alloc_gb": [
1064
+ 21.7,
1065
+ 21.72,
1066
+ 21.71,
1067
+ 21.71,
1068
+ 21.7,
1069
+ 21.7,
1070
+ 21.7,
1071
+ 21.72
1072
+ ],
1073
+ "max_mem_reserved_gb": [
1074
+ 22.04,
1075
+ 22.06,
1076
+ 22.06,
1077
+ 22.06,
1078
+ 22.06,
1079
+ 22.04,
1080
+ 22.05,
1081
+ 22.09
1082
+ ]
1083
+ },
1084
+ {
1085
+ "step": 2000,
1086
+ "loss": 3.2122890949249268,
1087
+ "lr": 0.0005314293900480271,
1088
+ "elapsed_s": 1506.8471064567566,
1089
+ "util": [
1090
+ 0.9365,
1091
+ 0.9932
1092
+ ],
1093
+ "drop": [
1094
+ 0.0,
1095
+ 0.0
1096
+ ],
1097
+ "traffic": [
1098
+ {
1099
+ "k": 3,
1100
+ "reads_keep1": 2.988037109375,
1101
+ "reads_keep3": 2.9285888671875
1102
+ },
1103
+ {
1104
+ "k": 3,
1105
+ "reads_keep1": 2.974609375,
1106
+ "reads_keep3": 2.9283447265625
1107
+ }
1108
+ ],
1109
+ "step_s": 0.7553,
1110
+ "grad_norm": 0.83733,
1111
+ "grad_norm_mean": 0.29064,
1112
+ "clip_frac": 0.02,
1113
+ "pairs_per_step": 786432.0,
1114
+ "max_mem_alloc_gb": [
1115
+ 21.7,
1116
+ 21.72,
1117
+ 21.71,
1118
+ 21.71,
1119
+ 21.7,
1120
+ 21.7,
1121
+ 21.7,
1122
+ 21.72
1123
+ ],
1124
+ "max_mem_reserved_gb": [
1125
+ 22.04,
1126
+ 22.06,
1127
+ 22.06,
1128
+ 22.06,
1129
+ 22.06,
1130
+ 22.04,
1131
+ 22.05,
1132
+ 22.09
1133
+ ]
1134
+ },
1135
+ {
1136
+ "step": 2100,
1137
+ "loss": 3.0985865592956543,
1138
+ "lr": 0.0005235397733506371,
1139
+ "elapsed_s": 1582.2666125297546,
1140
+ "util": [
1141
+ 0.9316,
1142
+ 0.9912
1143
+ ],
1144
+ "drop": [
1145
+ 0.0,
1146
+ 0.0
1147
+ ],
1148
+ "traffic": [
1149
+ {
1150
+ "k": 3,
1151
+ "reads_keep1": 2.990478515625,
1152
+ "reads_keep3": 2.9525146484375
1153
+ },
1154
+ {
1155
+ "k": 3,
1156
+ "reads_keep1": 2.9775390625,
1157
+ "reads_keep3": 2.9490966796875
1158
+ }
1159
+ ],
1160
+ "step_s": 0.7541,
1161
+ "grad_norm": 0.20209,
1162
+ "grad_norm_mean": 0.22126,
1163
+ "clip_frac": 0.0,
1164
+ "pairs_per_step": 786432.0,
1165
+ "max_mem_alloc_gb": [
1166
+ 21.7,
1167
+ 21.72,
1168
+ 21.71,
1169
+ 21.71,
1170
+ 21.7,
1171
+ 21.7,
1172
+ 21.7,
1173
+ 21.72
1174
+ ],
1175
+ "max_mem_reserved_gb": [
1176
+ 22.04,
1177
+ 22.06,
1178
+ 22.06,
1179
+ 22.06,
1180
+ 22.06,
1181
+ 22.04,
1182
+ 22.05,
1183
+ 22.09
1184
+ ]
1185
+ },
1186
+ {
1187
+ "step": 2200,
1188
+ "loss": 3.3588297367095947,
1189
+ "lr": 0.0005152945957257758,
1190
+ "elapsed_s": 1655.7793147563934,
1191
+ "util": [
1192
+ 0.9297,
1193
+ 0.9927
1194
+ ],
1195
+ "drop": [
1196
+ 0.0,
1197
+ 0.0
1198
+ ],
1199
+ "traffic": [
1200
+ {
1201
+ "k": 3,
1202
+ "reads_keep1": 2.9903564453125,
1203
+ "reads_keep3": 2.9365234375
1204
+ },
1205
+ {
1206
+ "k": 3,
1207
+ "reads_keep1": 2.976318359375,
1208
+ "reads_keep3": 2.937744140625
1209
+ }
1210
+ ],
1211
+ "step_s": 0.7352,
1212
+ "grad_norm": 0.2092,
1213
+ "grad_norm_mean": 0.2108,
1214
+ "clip_frac": 0.0,
1215
+ "pairs_per_step": 786432.0,
1216
+ "max_mem_alloc_gb": [
1217
+ 21.7,
1218
+ 21.72,
1219
+ 21.71,
1220
+ 21.71,
1221
+ 21.7,
1222
+ 21.7,
1223
+ 21.7,
1224
+ 21.72
1225
+ ],
1226
+ "max_mem_reserved_gb": [
1227
+ 22.04,
1228
+ 22.06,
1229
+ 22.06,
1230
+ 22.06,
1231
+ 22.06,
1232
+ 22.04,
1233
+ 22.05,
1234
+ 22.09
1235
+ ]
1236
+ },
1237
+ {
1238
+ "step": 2300,
1239
+ "loss": 3.402806282043457,
1240
+ "lr": 0.0005067090047730952,
1241
+ "elapsed_s": 1730.8209352493286,
1242
+ "util": [
1243
+ 0.9399,
1244
+ 0.9956
1245
+ ],
1246
+ "drop": [
1247
+ 0.0,
1248
+ 0.0
1249
+ ],
1250
+ "traffic": [
1251
+ {
1252
+ "k": 3,
1253
+ "reads_keep1": 2.9910888671875,
1254
+ "reads_keep3": 2.9541015625
1255
+ },
1256
+ {
1257
+ "k": 3,
1258
+ "reads_keep1": 2.9781494140625,
1259
+ "reads_keep3": 2.9521484375
1260
+ }
1261
+ ],
1262
+ "step_s": 0.7503,
1263
+ "grad_norm": 0.22093,
1264
+ "grad_norm_mean": 0.20946,
1265
+ "clip_frac": 0.0,
1266
+ "pairs_per_step": 786432.0,
1267
+ "max_mem_alloc_gb": [
1268
+ 21.7,
1269
+ 21.72,
1270
+ 21.71,
1271
+ 21.71,
1272
+ 21.7,
1273
+ 21.7,
1274
+ 21.7,
1275
+ 21.72
1276
+ ],
1277
+ "max_mem_reserved_gb": [
1278
+ 22.04,
1279
+ 22.06,
1280
+ 22.06,
1281
+ 22.06,
1282
+ 22.06,
1283
+ 22.04,
1284
+ 22.05,
1285
+ 22.09
1286
+ ]
1287
+ },
1288
+ {
1289
+ "step": 2400,
1290
+ "loss": 3.3327414989471436,
1291
+ "lr": 0.0004977987734814191,
1292
+ "elapsed_s": 1806.522236585617,
1293
+ "util": [
1294
+ 0.9351,
1295
+ 0.9971
1296
+ ],
1297
+ "drop": [
1298
+ 0.0,
1299
+ 0.0
1300
+ ],
1301
+ "traffic": [
1302
+ {
1303
+ "k": 3,
1304
+ "reads_keep1": 2.993408203125,
1305
+ "reads_keep3": 2.9530029296875
1306
+ },
1307
+ {
1308
+ "k": 3,
1309
+ "reads_keep1": 2.97705078125,
1310
+ "reads_keep3": 2.9482421875
1311
+ }
1312
+ ],
1313
+ "step_s": 0.757,
1314
+ "grad_norm": 0.22474,
1315
+ "grad_norm_mean": 0.21412,
1316
+ "clip_frac": 0.0,
1317
+ "pairs_per_step": 786432.0,
1318
+ "max_mem_alloc_gb": [
1319
+ 21.7,
1320
+ 21.72,
1321
+ 21.71,
1322
+ 21.71,
1323
+ 21.7,
1324
+ 21.7,
1325
+ 21.7,
1326
+ 21.72
1327
+ ],
1328
+ "max_mem_reserved_gb": [
1329
+ 22.04,
1330
+ 22.06,
1331
+ 22.06,
1332
+ 22.06,
1333
+ 22.06,
1334
+ 22.04,
1335
+ 22.05,
1336
+ 22.09
1337
+ ]
1338
+ },
1339
+ {
1340
+ "step": 2500,
1341
+ "loss": 2.9692678451538086,
1342
+ "lr": 0.0004885802712514513,
1343
+ "elapsed_s": 1882.090324163437,
1344
+ "util": [
1345
+ 0.9385,
1346
+ 0.9941
1347
+ ],
1348
+ "drop": [
1349
+ 0.0,
1350
+ 0.0
1351
+ ],
1352
+ "traffic": [
1353
+ {
1354
+ "k": 3,
1355
+ "reads_keep1": 2.9903564453125,
1356
+ "reads_keep3": 2.953369140625
1357
+ },
1358
+ {
1359
+ "k": 3,
1360
+ "reads_keep1": 2.978515625,
1361
+ "reads_keep3": 2.9488525390625
1362
+ }
1363
+ ],
1364
+ "step_s": 0.7557,
1365
+ "grad_norm": 0.20245,
1366
+ "grad_norm_mean": 0.20889,
1367
+ "clip_frac": 0.0,
1368
+ "pairs_per_step": 786432.0,
1369
+ "max_mem_alloc_gb": [
1370
+ 21.7,
1371
+ 21.72,
1372
+ 21.71,
1373
+ 21.71,
1374
+ 21.7,
1375
+ 21.7,
1376
+ 21.7,
1377
+ 21.72
1378
+ ],
1379
+ "max_mem_reserved_gb": [
1380
+ 22.04,
1381
+ 22.06,
1382
+ 22.06,
1383
+ 22.06,
1384
+ 22.06,
1385
+ 22.04,
1386
+ 22.05,
1387
+ 22.09
1388
+ ]
1389
+ },
1390
+ {
1391
+ "step": 2600,
1392
+ "loss": 3.281794548034668,
1393
+ "lr": 0.0004790704338227868,
1394
+ "elapsed_s": 1956.4616079330444,
1395
+ "util": [
1396
+ 0.9341,
1397
+ 0.9927
1398
+ ],
1399
+ "drop": [
1400
+ 0.0,
1401
+ 0.0
1402
+ ],
1403
+ "traffic": [
1404
+ {
1405
+ "k": 3,
1406
+ "reads_keep1": 2.994873046875,
1407
+ "reads_keep3": 2.9566650390625
1408
+ },
1409
+ {
1410
+ "k": 3,
1411
+ "reads_keep1": 2.97900390625,
1412
+ "reads_keep3": 2.9503173828125
1413
+ }
1414
+ ],
1415
+ "step_s": 0.7437,
1416
+ "grad_norm": 0.20784,
1417
+ "grad_norm_mean": 0.20703,
1418
+ "clip_frac": 0.0,
1419
+ "pairs_per_step": 786432.0,
1420
+ "max_mem_alloc_gb": [
1421
+ 21.7,
1422
+ 21.72,
1423
+ 21.71,
1424
+ 21.71,
1425
+ 21.7,
1426
+ 21.7,
1427
+ 21.7,
1428
+ 21.72
1429
+ ],
1430
+ "max_mem_reserved_gb": [
1431
+ 22.04,
1432
+ 22.06,
1433
+ 22.06,
1434
+ 22.06,
1435
+ 22.06,
1436
+ 22.04,
1437
+ 22.05,
1438
+ 22.09
1439
+ ]
1440
+ },
1441
+ {
1442
+ "step": 2700,
1443
+ "loss": 2.984389305114746,
1444
+ "lr": 0.0004692867321604769,
1445
+ "elapsed_s": 2030.7483987808228,
1446
+ "util": [
1447
+ 0.9453,
1448
+ 0.9956
1449
+ ],
1450
+ "drop": [
1451
+ 0.0,
1452
+ 0.0
1453
+ ],
1454
+ "traffic": [
1455
+ {
1456
+ "k": 3,
1457
+ "reads_keep1": 2.9844970703125,
1458
+ "reads_keep3": 2.9420166015625
1459
+ },
1460
+ {
1461
+ "k": 3,
1462
+ "reads_keep1": 2.965087890625,
1463
+ "reads_keep3": 2.9307861328125
1464
+ }
1465
+ ],
1466
+ "step_s": 0.7429,
1467
+ "grad_norm": 0.21195,
1468
+ "grad_norm_mean": 0.21187,
1469
+ "clip_frac": 0.0,
1470
+ "pairs_per_step": 786432.0,
1471
+ "max_mem_alloc_gb": [
1472
+ 21.7,
1473
+ 21.72,
1474
+ 21.71,
1475
+ 21.71,
1476
+ 21.7,
1477
+ 21.7,
1478
+ 21.7,
1479
+ 21.72
1480
+ ],
1481
+ "max_mem_reserved_gb": [
1482
+ 22.04,
1483
+ 22.06,
1484
+ 22.06,
1485
+ 22.06,
1486
+ 22.06,
1487
+ 22.04,
1488
+ 22.05,
1489
+ 22.09
1490
+ ]
1491
+ },
1492
+ {
1493
+ "step": 2800,
1494
+ "loss": 3.272160053253174,
1495
+ "lr": 0.0004592471403583057,
1496
+ "elapsed_s": 2104.6319546699524,
1497
+ "util": [
1498
+ 0.936,
1499
+ 0.9941
1500
+ ],
1501
+ "drop": [
1502
+ 0.0,
1503
+ 0.0
1504
+ ],
1505
+ "traffic": [
1506
+ {
1507
+ "k": 3,
1508
+ "reads_keep1": 2.993408203125,
1509
+ "reads_keep3": 2.9515380859375
1510
+ },
1511
+ {
1512
+ "k": 3,
1513
+ "reads_keep1": 2.982177734375,
1514
+ "reads_keep3": 2.950927734375
1515
+ }
1516
+ ],
1517
+ "step_s": 0.7389,
1518
+ "grad_norm": 0.20636,
1519
+ "grad_norm_mean": 0.21141,
1520
+ "clip_frac": 0.0,
1521
+ "pairs_per_step": 786432.0,
1522
+ "max_mem_alloc_gb": [
1523
+ 21.7,
1524
+ 21.72,
1525
+ 21.71,
1526
+ 21.71,
1527
+ 21.7,
1528
+ 21.7,
1529
+ 21.7,
1530
+ 21.72
1531
+ ],
1532
+ "max_mem_reserved_gb": [
1533
+ 22.04,
1534
+ 22.06,
1535
+ 22.06,
1536
+ 22.06,
1537
+ 22.06,
1538
+ 22.04,
1539
+ 22.05,
1540
+ 22.09
1541
+ ]
1542
+ },
1543
+ {
1544
+ "step": 2900,
1545
+ "loss": 3.1806206703186035,
1546
+ "lr": 0.00044897010261774577,
1547
+ "elapsed_s": 2178.3615646362305,
1548
+ "util": [
1549
+ 0.9458,
1550
+ 0.9971
1551
+ ],
1552
+ "drop": [
1553
+ 0.0,
1554
+ 0.0
1555
+ ],
1556
+ "traffic": [
1557
+ {
1558
+ "k": 3,
1559
+ "reads_keep1": 2.9932861328125,
1560
+ "reads_keep3": 2.95751953125
1561
+ },
1562
+ {
1563
+ "k": 3,
1564
+ "reads_keep1": 2.9788818359375,
1565
+ "reads_keep3": 2.9490966796875
1566
+ }
1567
+ ],
1568
+ "step_s": 0.7372,
1569
+ "grad_norm": 0.19491,
1570
+ "grad_norm_mean": 0.20595,
1571
+ "clip_frac": 0.0,
1572
+ "pairs_per_step": 786432.0,
1573
+ "max_mem_alloc_gb": [
1574
+ 21.7,
1575
+ 21.72,
1576
+ 21.71,
1577
+ 21.71,
1578
+ 21.7,
1579
+ 21.7,
1580
+ 21.7,
1581
+ 21.72
1582
+ ],
1583
+ "max_mem_reserved_gb": [
1584
+ 22.04,
1585
+ 22.06,
1586
+ 22.06,
1587
+ 22.06,
1588
+ 22.06,
1589
+ 22.04,
1590
+ 22.05,
1591
+ 22.09
1592
+ ]
1593
+ },
1594
+ {
1595
+ "step": 3000,
1596
+ "loss": 2.70481014251709,
1597
+ "lr": 0.00043847449936325653,
1598
+ "elapsed_s": 2253.2748243808746,
1599
+ "util": [
1600
+ 0.9492,
1601
+ 0.9971
1602
+ ],
1603
+ "drop": [
1604
+ 0.0,
1605
+ 0.0
1606
+ ],
1607
+ "traffic": [
1608
+ {
1609
+ "k": 3,
1610
+ "reads_keep1": 2.9908447265625,
1611
+ "reads_keep3": 2.954345703125
1612
+ },
1613
+ {
1614
+ "k": 3,
1615
+ "reads_keep1": 2.9769287109375,
1616
+ "reads_keep3": 2.947265625
1617
+ }
1618
+ ],
1619
+ "step_s": 0.7492,
1620
+ "grad_norm": 0.19955,
1621
+ "grad_norm_mean": 0.20256,
1622
+ "clip_frac": 0.0,
1623
+ "pairs_per_step": 786432.0,
1624
+ "max_mem_alloc_gb": [
1625
+ 21.7,
1626
+ 21.72,
1627
+ 21.71,
1628
+ 21.71,
1629
+ 21.7,
1630
+ 21.7,
1631
+ 21.7,
1632
+ 21.72
1633
+ ],
1634
+ "max_mem_reserved_gb": [
1635
+ 22.04,
1636
+ 22.06,
1637
+ 22.06,
1638
+ 22.06,
1639
+ 22.06,
1640
+ 22.04,
1641
+ 22.05,
1642
+ 22.09
1643
+ ]
1644
+ },
1645
+ {
1646
+ "step": 3100,
1647
+ "loss": 2.853893756866455,
1648
+ "lr": 0.00042777961255617835,
1649
+ "elapsed_s": 2325.9886758327484,
1650
+ "util": [
1651
+ 0.9351,
1652
+ 0.9951
1653
+ ],
1654
+ "drop": [
1655
+ 0.0,
1656
+ 0.0
1657
+ ],
1658
+ "traffic": [
1659
+ {
1660
+ "k": 3,
1661
+ "reads_keep1": 2.989501953125,
1662
+ "reads_keep3": 2.9544677734375
1663
+ },
1664
+ {
1665
+ "k": 3,
1666
+ "reads_keep1": 2.972900390625,
1667
+ "reads_keep3": 2.944580078125
1668
+ }
1669
+ ],
1670
+ "step_s": 0.7271,
1671
+ "grad_norm": 0.20644,
1672
+ "grad_norm_mean": 0.20348,
1673
+ "clip_frac": 0.0,
1674
+ "pairs_per_step": 786432.0,
1675
+ "max_mem_alloc_gb": [
1676
+ 21.7,
1677
+ 21.72,
1678
+ 21.71,
1679
+ 21.71,
1680
+ 21.7,
1681
+ 21.7,
1682
+ 21.7,
1683
+ 21.72
1684
+ ],
1685
+ "max_mem_reserved_gb": [
1686
+ 22.04,
1687
+ 22.06,
1688
+ 22.06,
1689
+ 22.06,
1690
+ 22.06,
1691
+ 22.04,
1692
+ 22.05,
1693
+ 22.09
1694
+ ]
1695
+ },
1696
+ {
1697
+ "step": 3200,
1698
+ "loss": 3.1224350929260254,
1699
+ "lr": 0.0004169050902709431,
1700
+ "elapsed_s": 2399.321888923645,
1701
+ "util": [
1702
+ 0.959,
1703
+ 0.9971
1704
+ ],
1705
+ "drop": [
1706
+ 0.0,
1707
+ 0.0
1708
+ ],
1709
+ "traffic": [
1710
+ {
1711
+ "k": 3,
1712
+ "reads_keep1": 2.990478515625,
1713
+ "reads_keep3": 2.95849609375
1714
+ },
1715
+ {
1716
+ "k": 3,
1717
+ "reads_keep1": 2.976318359375,
1718
+ "reads_keep3": 2.945556640625
1719
+ }
1720
+ ],
1721
+ "step_s": 0.7335,
1722
+ "grad_norm": 0.20256,
1723
+ "grad_norm_mean": 0.20802,
1724
+ "clip_frac": 0.0,
1725
+ "pairs_per_step": 786432.0,
1726
+ "max_mem_alloc_gb": [
1727
+ 21.7,
1728
+ 21.72,
1729
+ 21.71,
1730
+ 21.71,
1731
+ 21.7,
1732
+ 21.7,
1733
+ 21.7,
1734
+ 21.72
1735
+ ],
1736
+ "max_mem_reserved_gb": [
1737
+ 22.04,
1738
+ 22.06,
1739
+ 22.06,
1740
+ 22.06,
1741
+ 22.06,
1742
+ 22.04,
1743
+ 22.05,
1744
+ 22.09
1745
+ ]
1746
+ },
1747
+ {
1748
+ "step": 3300,
1749
+ "loss": 3.4007363319396973,
1750
+ "lr": 0.0004058709105986821,
1751
+ "elapsed_s": 2474.8059916496277,
1752
+ "util": [
1753
+ 0.9556,
1754
+ 0.9985
1755
+ ],
1756
+ "drop": [
1757
+ 0.0,
1758
+ 0.0
1759
+ ],
1760
+ "traffic": [
1761
+ {
1762
+ "k": 3,
1763
+ "reads_keep1": 2.99267578125,
1764
+ "reads_keep3": 2.9609375
1765
+ },
1766
+ {
1767
+ "k": 3,
1768
+ "reads_keep1": 2.981201171875,
1769
+ "reads_keep3": 2.9544677734375
1770
+ }
1771
+ ],
1772
+ "step_s": 0.7547,
1773
+ "grad_norm": 0.21276,
1774
+ "grad_norm_mean": 0.20332,
1775
+ "clip_frac": 0.0,
1776
+ "pairs_per_step": 786432.0,
1777
+ "max_mem_alloc_gb": [
1778
+ 21.7,
1779
+ 21.72,
1780
+ 21.71,
1781
+ 21.71,
1782
+ 21.7,
1783
+ 21.7,
1784
+ 21.7,
1785
+ 21.72
1786
+ ],
1787
+ "max_mem_reserved_gb": [
1788
+ 22.04,
1789
+ 22.06,
1790
+ 22.06,
1791
+ 22.06,
1792
+ 22.06,
1793
+ 22.04,
1794
+ 22.05,
1795
+ 22.09
1796
+ ]
1797
+ },
1798
+ {
1799
+ "step": 3400,
1800
+ "loss": 3.32659912109375,
1801
+ "lr": 0.000394697344944545,
1802
+ "elapsed_s": 2549.775881290436,
1803
+ "util": [
1804
+ 0.9478,
1805
+ 0.9946
1806
+ ],
1807
+ "drop": [
1808
+ 0.0,
1809
+ 0.0
1810
+ ],
1811
+ "traffic": [
1812
+ {
1813
+ "k": 3,
1814
+ "reads_keep1": 2.9892578125,
1815
+ "reads_keep3": 2.9425048828125
1816
+ },
1817
+ {
1818
+ "k": 3,
1819
+ "reads_keep1": 2.974365234375,
1820
+ "reads_keep3": 2.9466552734375
1821
+ }
1822
+ ],
1823
+ "step_s": 0.7497,
1824
+ "grad_norm": 0.20353,
1825
+ "grad_norm_mean": 0.20228,
1826
+ "clip_frac": 0.0,
1827
+ "pairs_per_step": 786432.0,
1828
+ "max_mem_alloc_gb": [
1829
+ 21.7,
1830
+ 21.72,
1831
+ 21.71,
1832
+ 21.71,
1833
+ 21.7,
1834
+ 21.7,
1835
+ 21.7,
1836
+ 21.72
1837
+ ],
1838
+ "max_mem_reserved_gb": [
1839
+ 22.04,
1840
+ 22.06,
1841
+ 22.06,
1842
+ 22.06,
1843
+ 22.06,
1844
+ 22.04,
1845
+ 22.05,
1846
+ 22.09
1847
+ ]
1848
+ },
1849
+ {
1850
+ "step": 3500,
1851
+ "loss": 2.9369020462036133,
1852
+ "lr": 0.0003834049207861575,
1853
+ "elapsed_s": 2623.84023809433,
1854
+ "util": [
1855
+ 0.9585,
1856
+ 0.9937
1857
+ ],
1858
+ "drop": [
1859
+ 0.0,
1860
+ 0.0
1861
+ ],
1862
+ "traffic": [
1863
+ {
1864
+ "k": 3,
1865
+ "reads_keep1": 2.9920654296875,
1866
+ "reads_keep3": 2.9344482421875
1867
+ },
1868
+ {
1869
+ "k": 3,
1870
+ "reads_keep1": 2.975341796875,
1871
+ "reads_keep3": 2.93359375
1872
+ }
1873
+ ],
1874
+ "step_s": 0.7407,
1875
+ "grad_norm": 0.19558,
1876
+ "grad_norm_mean": 0.20182,
1877
+ "clip_frac": 0.0,
1878
+ "pairs_per_step": 786432.0,
1879
+ "max_mem_alloc_gb": [
1880
+ 21.7,
1881
+ 21.72,
1882
+ 21.71,
1883
+ 21.71,
1884
+ 21.7,
1885
+ 21.7,
1886
+ 21.7,
1887
+ 21.72
1888
+ ],
1889
+ "max_mem_reserved_gb": [
1890
+ 22.04,
1891
+ 22.06,
1892
+ 22.06,
1893
+ 22.06,
1894
+ 22.06,
1895
+ 22.04,
1896
+ 22.05,
1897
+ 22.09
1898
+ ]
1899
+ },
1900
+ {
1901
+ "step": 3600,
1902
+ "loss": 3.1533007621765137,
1903
+ "lr": 0.0003720143839616368,
1904
+ "elapsed_s": 2698.953490257263,
1905
+ "util": [
1906
+ 0.96,
1907
+ 0.9961
1908
+ ],
1909
+ "drop": [
1910
+ 0.0,
1911
+ 0.0
1912
+ ],
1913
+ "traffic": [
1914
+ {
1915
+ "k": 3,
1916
+ "reads_keep1": 2.9942626953125,
1917
+ "reads_keep3": 2.9639892578125
1918
+ },
1919
+ {
1920
+ "k": 3,
1921
+ "reads_keep1": 2.9796142578125,
1922
+ "reads_keep3": 2.9532470703125
1923
+ }
1924
+ ],
1925
+ "step_s": 0.751,
1926
+ "grad_norm": 0.19066,
1927
+ "grad_norm_mean": 0.20188,
1928
+ "clip_frac": 0.0,
1929
+ "pairs_per_step": 786432.0,
1930
+ "max_mem_alloc_gb": [
1931
+ 21.7,
1932
+ 21.72,
1933
+ 21.71,
1934
+ 21.71,
1935
+ 21.7,
1936
+ 21.7,
1937
+ 21.7,
1938
+ 21.72
1939
+ ],
1940
+ "max_mem_reserved_gb": [
1941
+ 22.04,
1942
+ 22.06,
1943
+ 22.06,
1944
+ 22.06,
1945
+ 22.06,
1946
+ 22.04,
1947
+ 22.05,
1948
+ 22.09
1949
+ ]
1950
+ },
1951
+ {
1952
+ "step": 3700,
1953
+ "loss": 2.912728786468506,
1954
+ "lr": 0.0003605466605564464,
1955
+ "elapsed_s": 2773.2590370178223,
1956
+ "util": [
1957
+ 0.9551,
1958
+ 0.999
1959
+ ],
1960
+ "drop": [
1961
+ 0.0,
1962
+ 0.0
1963
+ ],
1964
+ "traffic": [
1965
+ {
1966
+ "k": 3,
1967
+ "reads_keep1": 2.99072265625,
1968
+ "reads_keep3": 2.959228515625
1969
+ },
1970
+ {
1971
+ "k": 3,
1972
+ "reads_keep1": 2.96826171875,
1973
+ "reads_keep3": 2.93994140625
1974
+ }
1975
+ ],
1976
+ "step_s": 0.7431,
1977
+ "grad_norm": 0.21287,
1978
+ "grad_norm_mean": 0.19817,
1979
+ "clip_frac": 0.0,
1980
+ "pairs_per_step": 786432.0,
1981
+ "max_mem_alloc_gb": [
1982
+ 21.7,
1983
+ 21.72,
1984
+ 21.71,
1985
+ 21.71,
1986
+ 21.7,
1987
+ 21.7,
1988
+ 21.7,
1989
+ 21.72
1990
+ ],
1991
+ "max_mem_reserved_gb": [
1992
+ 22.04,
1993
+ 22.06,
1994
+ 22.06,
1995
+ 22.06,
1996
+ 22.06,
1997
+ 22.04,
1998
+ 22.05,
1999
+ 22.09
2000
+ ]
2001
+ },
2002
+ {
2003
+ "step": 3800,
2004
+ "loss": 2.966920852661133,
2005
+ "lr": 0.0003490228184591104,
2006
+ "elapsed_s": 2847.961940050125,
2007
+ "util": [
2008
+ 0.9492,
2009
+ 0.9971
2010
+ ],
2011
+ "drop": [
2012
+ 0.0,
2013
+ 0.0
2014
+ ],
2015
+ "traffic": [
2016
+ {
2017
+ "k": 3,
2018
+ "reads_keep1": 2.9892578125,
2019
+ "reads_keep3": 2.9473876953125
2020
+ },
2021
+ {
2022
+ "k": 3,
2023
+ "reads_keep1": 2.96923828125,
2024
+ "reads_keep3": 2.9351806640625
2025
+ }
2026
+ ],
2027
+ "step_s": 0.747,
2028
+ "grad_norm": 0.20476,
2029
+ "grad_norm_mean": 0.19961,
2030
+ "clip_frac": 0.0,
2031
+ "pairs_per_step": 786432.0,
2032
+ "max_mem_alloc_gb": [
2033
+ 21.7,
2034
+ 21.72,
2035
+ 21.71,
2036
+ 21.71,
2037
+ 21.7,
2038
+ 21.7,
2039
+ 21.7,
2040
+ 21.72
2041
+ ],
2042
+ "max_mem_reserved_gb": [
2043
+ 22.04,
2044
+ 22.06,
2045
+ 22.06,
2046
+ 22.06,
2047
+ 22.06,
2048
+ 22.04,
2049
+ 22.05,
2050
+ 22.09
2051
+ ]
2052
+ },
2053
+ {
2054
+ "step": 3900,
2055
+ "loss": 2.9563255310058594,
2056
+ "lr": 0.00033746402865641464,
2057
+ "elapsed_s": 2922.7830786705017,
2058
+ "util": [
2059
+ 0.9678,
2060
+ 0.999
2061
+ ],
2062
+ "drop": [
2063
+ 0.0,
2064
+ 0.0
2065
+ ],
2066
+ "traffic": [
2067
+ {
2068
+ "k": 3,
2069
+ "reads_keep1": 2.9906005859375,
2070
+ "reads_keep3": 2.95068359375
2071
+ },
2072
+ {
2073
+ "k": 3,
2074
+ "reads_keep1": 2.965087890625,
2075
+ "reads_keep3": 2.935546875
2076
+ }
2077
+ ],
2078
+ "step_s": 0.7483,
2079
+ "grad_norm": 0.19311,
2080
+ "grad_norm_mean": 0.2005,
2081
+ "clip_frac": 0.0,
2082
+ "pairs_per_step": 786432.0,
2083
+ "max_mem_alloc_gb": [
2084
+ 21.7,
2085
+ 21.72,
2086
+ 21.71,
2087
+ 21.71,
2088
+ 21.7,
2089
+ 21.7,
2090
+ 21.7,
2091
+ 21.72
2092
+ ],
2093
+ "max_mem_reserved_gb": [
2094
+ 22.04,
2095
+ 22.06,
2096
+ 22.06,
2097
+ 22.06,
2098
+ 22.06,
2099
+ 22.04,
2100
+ 22.05,
2101
+ 22.09
2102
+ ]
2103
+ },
2104
+ {
2105
+ "step": 4000,
2106
+ "loss": 2.939749240875244,
2107
+ "lr": 0.0003258915263392015,
2108
+ "elapsed_s": 2998.234353542328,
2109
+ "util": [
2110
+ 0.96,
2111
+ 0.9971
2112
+ ],
2113
+ "drop": [
2114
+ 0.0,
2115
+ 0.0
2116
+ ],
2117
+ "traffic": [
2118
+ {
2119
+ "k": 3,
2120
+ "reads_keep1": 2.990478515625,
2121
+ "reads_keep3": 2.9595947265625
2122
+ },
2123
+ {
2124
+ "k": 3,
2125
+ "reads_keep1": 2.9766845703125,
2126
+ "reads_keep3": 2.9468994140625
2127
+ }
2128
+ ],
2129
+ "step_s": 0.7545,
2130
+ "grad_norm": 0.19522,
2131
+ "grad_norm_mean": 0.20158,
2132
+ "clip_frac": 0.0,
2133
+ "pairs_per_step": 786432.0,
2134
+ "max_mem_alloc_gb": [
2135
+ 21.7,
2136
+ 21.72,
2137
+ 21.71,
2138
+ 21.71,
2139
+ 21.7,
2140
+ 21.7,
2141
+ 21.7,
2142
+ 21.72
2143
+ ],
2144
+ "max_mem_reserved_gb": [
2145
+ 22.04,
2146
+ 22.06,
2147
+ 22.06,
2148
+ 22.06,
2149
+ 22.06,
2150
+ 22.04,
2151
+ 22.05,
2152
+ 22.09
2153
+ ]
2154
+ },
2155
+ {
2156
+ "step": 4100,
2157
+ "loss": 2.956996440887451,
2158
+ "lr": 0.00031432657189021235,
2159
+ "elapsed_s": 3073.1621329784393,
2160
+ "util": [
2161
+ 0.9707,
2162
+ 0.9985
2163
+ ],
2164
+ "drop": [
2165
+ 0.0,
2166
+ 0.0
2167
+ ],
2168
+ "traffic": [
2169
+ {
2170
+ "k": 3,
2171
+ "reads_keep1": 2.9903564453125,
2172
+ "reads_keep3": 2.95654296875
2173
+ },
2174
+ {
2175
+ "k": 3,
2176
+ "reads_keep1": 2.97265625,
2177
+ "reads_keep3": 2.9456787109375
2178
+ }
2179
+ ],
2180
+ "step_s": 0.7493,
2181
+ "grad_norm": 0.19031,
2182
+ "grad_norm_mean": 0.20092,
2183
+ "clip_frac": 0.0,
2184
+ "pairs_per_step": 786432.0,
2185
+ "max_mem_alloc_gb": [
2186
+ 21.7,
2187
+ 21.72,
2188
+ 21.71,
2189
+ 21.71,
2190
+ 21.7,
2191
+ 21.7,
2192
+ 21.7,
2193
+ 21.72
2194
+ ],
2195
+ "max_mem_reserved_gb": [
2196
+ 22.04,
2197
+ 22.06,
2198
+ 22.06,
2199
+ 22.06,
2200
+ 22.06,
2201
+ 22.04,
2202
+ 22.05,
2203
+ 22.09
2204
+ ]
2205
+ },
2206
+ {
2207
+ "step": 4200,
2208
+ "loss": 2.7563636302948,
2209
+ "lr": 0.00030279041182564807,
2210
+ "elapsed_s": 3149.9751269817352,
2211
+ "util": [
2212
+ 0.9487,
2213
+ 0.9917
2214
+ ],
2215
+ "drop": [
2216
+ 0.0,
2217
+ 0.0
2218
+ ],
2219
+ "traffic": [
2220
+ {
2221
+ "k": 3,
2222
+ "reads_keep1": 2.994384765625,
2223
+ "reads_keep3": 2.95458984375
2224
+ },
2225
+ {
2226
+ "k": 3,
2227
+ "reads_keep1": 2.9769287109375,
2228
+ "reads_keep3": 2.950439453125
2229
+ }
2230
+ ],
2231
+ "step_s": 0.7681,
2232
+ "grad_norm": 0.19292,
2233
+ "grad_norm_mean": 0.20121,
2234
+ "clip_frac": 0.0,
2235
+ "pairs_per_step": 786432.0,
2236
+ "max_mem_alloc_gb": [
2237
+ 21.7,
2238
+ 21.72,
2239
+ 21.71,
2240
+ 21.71,
2241
+ 21.7,
2242
+ 21.7,
2243
+ 21.7,
2244
+ 21.72
2245
+ ],
2246
+ "max_mem_reserved_gb": [
2247
+ 22.04,
2248
+ 22.06,
2249
+ 22.06,
2250
+ 22.06,
2251
+ 22.06,
2252
+ 22.04,
2253
+ 22.05,
2254
+ 22.09
2255
+ ]
2256
+ },
2257
+ {
2258
+ "step": 4300,
2259
+ "loss": 2.962275981903076,
2260
+ "lr": 0.0002913042397622058,
2261
+ "elapsed_s": 3225.034797668457,
2262
+ "util": [
2263
+ 0.9727,
2264
+ 0.9976
2265
+ ],
2266
+ "drop": [
2267
+ 0.0,
2268
+ 0.0
2269
+ ],
2270
+ "traffic": [
2271
+ {
2272
+ "k": 3,
2273
+ "reads_keep1": 2.99072265625,
2274
+ "reads_keep3": 2.9556884765625
2275
+ },
2276
+ {
2277
+ "k": 3,
2278
+ "reads_keep1": 2.97509765625,
2279
+ "reads_keep3": 2.9483642578125
2280
+ }
2281
+ ],
2282
+ "step_s": 0.7506,
2283
+ "grad_norm": 0.19897,
2284
+ "grad_norm_mean": 0.19914,
2285
+ "clip_frac": 0.0,
2286
+ "pairs_per_step": 786432.0,
2287
+ "max_mem_alloc_gb": [
2288
+ 21.7,
2289
+ 21.72,
2290
+ 21.71,
2291
+ 21.71,
2292
+ 21.7,
2293
+ 21.7,
2294
+ 21.7,
2295
+ 21.72
2296
+ ],
2297
+ "max_mem_reserved_gb": [
2298
+ 22.04,
2299
+ 22.06,
2300
+ 22.06,
2301
+ 22.06,
2302
+ 22.06,
2303
+ 22.04,
2304
+ 22.05,
2305
+ 22.09
2306
+ ]
2307
+ },
2308
+ {
2309
+ "step": 4400,
2310
+ "loss": 2.7788233757019043,
2311
+ "lr": 0.00027988915748129826,
2312
+ "elapsed_s": 3301.5959877967834,
2313
+ "util": [
2314
+ 0.957,
2315
+ 0.9937
2316
+ ],
2317
+ "drop": [
2318
+ 0.0,
2319
+ 0.0
2320
+ ],
2321
+ "traffic": [
2322
+ {
2323
+ "k": 3,
2324
+ "reads_keep1": 2.987548828125,
2325
+ "reads_keep3": 2.9591064453125
2326
+ },
2327
+ {
2328
+ "k": 3,
2329
+ "reads_keep1": 2.96875,
2330
+ "reads_keep3": 2.9462890625
2331
+ }
2332
+ ],
2333
+ "step_s": 0.7656,
2334
+ "grad_norm": 0.20864,
2335
+ "grad_norm_mean": 0.20094,
2336
+ "clip_frac": 0.0,
2337
+ "pairs_per_step": 786432.0,
2338
+ "max_mem_alloc_gb": [
2339
+ 21.7,
2340
+ 21.72,
2341
+ 21.71,
2342
+ 21.71,
2343
+ 21.7,
2344
+ 21.7,
2345
+ 21.7,
2346
+ 21.72
2347
+ ],
2348
+ "max_mem_reserved_gb": [
2349
+ 22.04,
2350
+ 22.06,
2351
+ 22.06,
2352
+ 22.06,
2353
+ 22.06,
2354
+ 22.04,
2355
+ 22.05,
2356
+ 22.09
2357
+ ]
2358
+ },
2359
+ {
2360
+ "step": 4500,
2361
+ "loss": 3.030665874481201,
2362
+ "lr": 0.0002685661361619898,
2363
+ "elapsed_s": 3378.687537908554,
2364
+ "util": [
2365
+ 0.96,
2366
+ 0.9976
2367
+ ],
2368
+ "drop": [
2369
+ 0.0,
2370
+ 0.0
2371
+ ],
2372
+ "traffic": [
2373
+ {
2374
+ "k": 3,
2375
+ "reads_keep1": 2.99462890625,
2376
+ "reads_keep3": 2.9603271484375
2377
+ },
2378
+ {
2379
+ "k": 3,
2380
+ "reads_keep1": 2.97509765625,
2381
+ "reads_keep3": 2.94580078125
2382
+ }
2383
+ ],
2384
+ "step_s": 0.771,
2385
+ "grad_norm": 0.20155,
2386
+ "grad_norm_mean": 0.20009,
2387
+ "clip_frac": 0.0,
2388
+ "pairs_per_step": 786432.0,
2389
+ "max_mem_alloc_gb": [
2390
+ 21.7,
2391
+ 21.72,
2392
+ 21.71,
2393
+ 21.71,
2394
+ 21.7,
2395
+ 21.7,
2396
+ 21.7,
2397
+ 21.72
2398
+ ],
2399
+ "max_mem_reserved_gb": [
2400
+ 22.04,
2401
+ 22.06,
2402
+ 22.06,
2403
+ 22.06,
2404
+ 22.06,
2405
+ 22.04,
2406
+ 22.05,
2407
+ 22.09
2408
+ ]
2409
+ },
2410
+ {
2411
+ "step": 4600,
2412
+ "loss": 2.576869010925293,
2413
+ "lr": 0.0002573559778538662,
2414
+ "elapsed_s": 3455.1875426769257,
2415
+ "util": [
2416
+ 0.9663,
2417
+ 0.9946
2418
+ ],
2419
+ "drop": [
2420
+ 0.0,
2421
+ 0.0
2422
+ ],
2423
+ "traffic": [
2424
+ {
2425
+ "k": 3,
2426
+ "reads_keep1": 2.9891357421875,
2427
+ "reads_keep3": 2.9534912109375
2428
+ },
2429
+ {
2430
+ "k": 3,
2431
+ "reads_keep1": 2.9708251953125,
2432
+ "reads_keep3": 2.9390869140625
2433
+ }
2434
+ ],
2435
+ "step_s": 0.765,
2436
+ "grad_norm": 0.19646,
2437
+ "grad_norm_mean": 0.20054,
2438
+ "clip_frac": 0.0,
2439
+ "pairs_per_step": 786432.0,
2440
+ "max_mem_alloc_gb": [
2441
+ 21.7,
2442
+ 21.72,
2443
+ 21.71,
2444
+ 21.71,
2445
+ 21.7,
2446
+ 21.7,
2447
+ 21.7,
2448
+ 21.72
2449
+ ],
2450
+ "max_mem_reserved_gb": [
2451
+ 22.04,
2452
+ 22.06,
2453
+ 22.06,
2454
+ 22.06,
2455
+ 22.06,
2456
+ 22.04,
2457
+ 22.05,
2458
+ 22.09
2459
+ ]
2460
+ },
2461
+ {
2462
+ "step": 4700,
2463
+ "loss": 2.840919256210327,
2464
+ "lr": 0.0002462792772606214,
2465
+ "elapsed_s": 3532.6897184848785,
2466
+ "util": [
2467
+ 0.9648,
2468
+ 0.9927
2469
+ ],
2470
+ "drop": [
2471
+ 0.0,
2472
+ 0.0
2473
+ ],
2474
+ "traffic": [
2475
+ {
2476
+ "k": 3,
2477
+ "reads_keep1": 2.99462890625,
2478
+ "reads_keep3": 2.9674072265625
2479
+ },
2480
+ {
2481
+ "k": 3,
2482
+ "reads_keep1": 2.973388671875,
2483
+ "reads_keep3": 2.94580078125
2484
+ }
2485
+ ],
2486
+ "step_s": 0.775,
2487
+ "grad_norm": 0.20917,
2488
+ "grad_norm_mean": 0.19826,
2489
+ "clip_frac": 0.0,
2490
+ "pairs_per_step": 786432.0,
2491
+ "max_mem_alloc_gb": [
2492
+ 21.7,
2493
+ 21.72,
2494
+ 21.71,
2495
+ 21.71,
2496
+ 21.7,
2497
+ 21.7,
2498
+ 21.7,
2499
+ 21.72
2500
+ ],
2501
+ "max_mem_reserved_gb": [
2502
+ 22.04,
2503
+ 22.06,
2504
+ 22.06,
2505
+ 22.06,
2506
+ 22.06,
2507
+ 22.04,
2508
+ 22.05,
2509
+ 22.09
2510
+ ]
2511
+ },
2512
+ {
2513
+ "step": 4800,
2514
+ "loss": 2.814150810241699,
2515
+ "lr": 0.00023535638390456822,
2516
+ "elapsed_s": 3611.532378435135,
2517
+ "util": [
2518
+ 0.9707,
2519
+ 0.999
2520
+ ],
2521
+ "drop": [
2522
+ 0.0,
2523
+ 0.0
2524
+ ],
2525
+ "traffic": [
2526
+ {
2527
+ "k": 3,
2528
+ "reads_keep1": 2.995849609375,
2529
+ "reads_keep3": 2.9617919921875
2530
+ },
2531
+ {
2532
+ "k": 3,
2533
+ "reads_keep1": 2.972900390625,
2534
+ "reads_keep3": 2.9423828125
2535
+ }
2536
+ ],
2537
+ "step_s": 0.7884,
2538
+ "grad_norm": 0.1892,
2539
+ "grad_norm_mean": 0.19727,
2540
+ "clip_frac": 0.0,
2541
+ "pairs_per_step": 786432.0,
2542
+ "max_mem_alloc_gb": [
2543
+ 21.7,
2544
+ 21.72,
2545
+ 21.71,
2546
+ 21.71,
2547
+ 21.7,
2548
+ 21.7,
2549
+ 21.7,
2550
+ 21.72
2551
+ ],
2552
+ "max_mem_reserved_gb": [
2553
+ 22.04,
2554
+ 22.06,
2555
+ 22.06,
2556
+ 22.06,
2557
+ 22.06,
2558
+ 22.04,
2559
+ 22.05,
2560
+ 22.09
2561
+ ]
2562
+ },
2563
+ {
2564
+ "step": 4900,
2565
+ "loss": 2.9387612342834473,
2566
+ "lr": 0.0002246073647415842,
2567
+ "elapsed_s": 3690.9922754764557,
2568
+ "util": [
2569
+ 0.9639,
2570
+ 0.9937
2571
+ ],
2572
+ "drop": [
2573
+ 0.0,
2574
+ 0.0
2575
+ ],
2576
+ "traffic": [
2577
+ {
2578
+ "k": 3,
2579
+ "reads_keep1": 2.9947509765625,
2580
+ "reads_keep3": 2.95654296875
2581
+ },
2582
+ {
2583
+ "k": 3,
2584
+ "reads_keep1": 2.9759521484375,
2585
+ "reads_keep3": 2.9425048828125
2586
+ }
2587
+ ],
2588
+ "step_s": 0.7947,
2589
+ "grad_norm": 0.19499,
2590
+ "grad_norm_mean": 0.19865,
2591
+ "clip_frac": 0.0,
2592
+ "pairs_per_step": 786432.0,
2593
+ "max_mem_alloc_gb": [
2594
+ 21.7,
2595
+ 21.72,
2596
+ 21.71,
2597
+ 21.71,
2598
+ 21.7,
2599
+ 21.7,
2600
+ 21.7,
2601
+ 21.72
2602
+ ],
2603
+ "max_mem_reserved_gb": [
2604
+ 22.04,
2605
+ 22.06,
2606
+ 22.06,
2607
+ 22.06,
2608
+ 22.06,
2609
+ 22.04,
2610
+ 22.05,
2611
+ 22.09
2612
+ ]
2613
+ },
2614
+ {
2615
+ "step": 5000,
2616
+ "loss": 3.032792568206787,
2617
+ "lr": 0.00021405196729517236,
2618
+ "elapsed_s": 3770.0510654449463,
2619
+ "util": [
2620
+ 0.9751,
2621
+ 0.9995
2622
+ ],
2623
+ "drop": [
2624
+ 0.0,
2625
+ 0.0
2626
+ ],
2627
+ "traffic": [
2628
+ {
2629
+ "k": 3,
2630
+ "reads_keep1": 2.9921875,
2631
+ "reads_keep3": 2.9609375
2632
+ },
2633
+ {
2634
+ "k": 3,
2635
+ "reads_keep1": 2.975830078125,
2636
+ "reads_keep3": 2.9495849609375
2637
+ }
2638
+ ],
2639
+ "step_s": 0.7906,
2640
+ "grad_norm": 0.20141,
2641
+ "grad_norm_mean": 0.19843,
2642
+ "clip_frac": 0.0,
2643
+ "pairs_per_step": 786432.0,
2644
+ "max_mem_alloc_gb": [
2645
+ 21.7,
2646
+ 21.72,
2647
+ 21.71,
2648
+ 21.71,
2649
+ 21.7,
2650
+ 21.7,
2651
+ 21.7,
2652
+ 21.72
2653
+ ],
2654
+ "max_mem_reserved_gb": [
2655
+ 22.04,
2656
+ 22.06,
2657
+ 22.06,
2658
+ 22.06,
2659
+ 22.06,
2660
+ 22.04,
2661
+ 22.05,
2662
+ 22.09
2663
+ ]
2664
+ },
2665
+ {
2666
+ "step": 5100,
2667
+ "loss": 2.979060173034668,
2668
+ "lr": 0.00020370958337736627,
2669
+ "elapsed_s": 3849.664798259735,
2670
+ "util": [
2671
+ 0.978,
2672
+ 0.9976
2673
+ ],
2674
+ "drop": [
2675
+ 0.0,
2676
+ 0.0
2677
+ ],
2678
+ "traffic": [
2679
+ {
2680
+ "k": 3,
2681
+ "reads_keep1": 2.990478515625,
2682
+ "reads_keep3": 2.9471435546875
2683
+ },
2684
+ {
2685
+ "k": 3,
2686
+ "reads_keep1": 2.974609375,
2687
+ "reads_keep3": 2.9388427734375
2688
+ }
2689
+ ],
2690
+ "step_s": 0.7962,
2691
+ "grad_norm": 0.18802,
2692
+ "grad_norm_mean": 0.19778,
2693
+ "clip_frac": 0.0,
2694
+ "pairs_per_step": 786432.0,
2695
+ "max_mem_alloc_gb": [
2696
+ 21.7,
2697
+ 21.72,
2698
+ 21.71,
2699
+ 21.71,
2700
+ 21.7,
2701
+ 21.7,
2702
+ 21.7,
2703
+ 21.72
2704
+ ],
2705
+ "max_mem_reserved_gb": [
2706
+ 22.04,
2707
+ 22.06,
2708
+ 22.06,
2709
+ 22.06,
2710
+ 22.06,
2711
+ 22.04,
2712
+ 22.05,
2713
+ 22.09
2714
+ ]
2715
+ },
2716
+ {
2717
+ "step": 5200,
2718
+ "loss": 3.0355381965637207,
2719
+ "lr": 0.00019359921346313,
2720
+ "elapsed_s": 3929.0365781784058,
2721
+ "util": [
2722
+ 0.9697,
2723
+ 0.9976
2724
+ ],
2725
+ "drop": [
2726
+ 0.0,
2727
+ 0.0
2728
+ ],
2729
+ "traffic": [
2730
+ {
2731
+ "k": 3,
2732
+ "reads_keep1": 2.99365234375,
2733
+ "reads_keep3": 2.967529296875
2734
+ },
2735
+ {
2736
+ "k": 3,
2737
+ "reads_keep1": 2.9744873046875,
2738
+ "reads_keep3": 2.9498291015625
2739
+ }
2740
+ ],
2741
+ "step_s": 0.7937,
2742
+ "grad_norm": 0.19806,
2743
+ "grad_norm_mean": 0.19832,
2744
+ "clip_frac": 0.0,
2745
+ "pairs_per_step": 786432.0,
2746
+ "max_mem_alloc_gb": [
2747
+ 21.7,
2748
+ 21.72,
2749
+ 21.71,
2750
+ 21.71,
2751
+ 21.7,
2752
+ 21.7,
2753
+ 21.7,
2754
+ 21.72
2755
+ ],
2756
+ "max_mem_reserved_gb": [
2757
+ 22.04,
2758
+ 22.06,
2759
+ 22.06,
2760
+ 22.06,
2761
+ 22.06,
2762
+ 22.04,
2763
+ 22.05,
2764
+ 22.09
2765
+ ]
2766
+ },
2767
+ {
2768
+ "step": 5300,
2769
+ "loss": 3.0029296875,
2770
+ "lr": 0.00018373943178370066,
2771
+ "elapsed_s": 4008.2700369358063,
2772
+ "util": [
2773
+ 0.9819,
2774
+ 1.0
2775
+ ],
2776
+ "drop": [
2777
+ 0.0,
2778
+ 0.0
2779
+ ],
2780
+ "traffic": [
2781
+ {
2782
+ "k": 3,
2783
+ "reads_keep1": 2.99365234375,
2784
+ "reads_keep3": 2.970947265625
2785
+ },
2786
+ {
2787
+ "k": 3,
2788
+ "reads_keep1": 2.9755859375,
2789
+ "reads_keep3": 2.9503173828125
2790
+ }
2791
+ ],
2792
+ "step_s": 0.7923,
2793
+ "grad_norm": 0.19581,
2794
+ "grad_norm_mean": 0.19797,
2795
+ "clip_frac": 0.0,
2796
+ "pairs_per_step": 786432.0,
2797
+ "max_mem_alloc_gb": [
2798
+ 21.7,
2799
+ 21.72,
2800
+ 21.71,
2801
+ 21.71,
2802
+ 21.7,
2803
+ 21.7,
2804
+ 21.7,
2805
+ 21.72
2806
+ ],
2807
+ "max_mem_reserved_gb": [
2808
+ 22.04,
2809
+ 22.06,
2810
+ 22.06,
2811
+ 22.06,
2812
+ 22.06,
2813
+ 22.04,
2814
+ 22.05,
2815
+ 22.09
2816
+ ]
2817
+ },
2818
+ {
2819
+ "step": 5400,
2820
+ "loss": 3.0728814601898193,
2821
+ "lr": 0.00017414835220300347,
2822
+ "elapsed_s": 4088.0783507823944,
2823
+ "util": [
2824
+ 0.9766,
2825
+ 0.9966
2826
+ ],
2827
+ "drop": [
2828
+ 0.0,
2829
+ 0.0
2830
+ ],
2831
+ "traffic": [
2832
+ {
2833
+ "k": 3,
2834
+ "reads_keep1": 2.99462890625,
2835
+ "reads_keep3": 2.970458984375
2836
+ },
2837
+ {
2838
+ "k": 3,
2839
+ "reads_keep1": 2.97119140625,
2840
+ "reads_keep3": 2.9466552734375
2841
+ }
2842
+ ],
2843
+ "step_s": 0.7981,
2844
+ "grad_norm": 0.19208,
2845
+ "grad_norm_mean": 0.1978,
2846
+ "clip_frac": 0.0,
2847
+ "pairs_per_step": 786432.0,
2848
+ "max_mem_alloc_gb": [
2849
+ 21.7,
2850
+ 21.72,
2851
+ 21.71,
2852
+ 21.71,
2853
+ 21.7,
2854
+ 21.7,
2855
+ 21.7,
2856
+ 21.72
2857
+ ],
2858
+ "max_mem_reserved_gb": [
2859
+ 22.04,
2860
+ 22.06,
2861
+ 22.06,
2862
+ 22.06,
2863
+ 22.06,
2864
+ 22.04,
2865
+ 22.05,
2866
+ 22.09
2867
+ ]
2868
+ },
2869
+ {
2870
+ "step": 5500,
2871
+ "loss": 2.8215982913970947,
2872
+ "lr": 0.0001648435949398293,
2873
+ "elapsed_s": 4171.449336767197,
2874
+ "util": [
2875
+ 0.9775,
2876
+ 0.998
2877
+ ],
2878
+ "drop": [
2879
+ 0.0,
2880
+ 0.0
2881
+ ],
2882
+ "traffic": [
2883
+ {
2884
+ "k": 3,
2885
+ "reads_keep1": 2.9835205078125,
2886
+ "reads_keep3": 2.92138671875
2887
+ },
2888
+ {
2889
+ "k": 3,
2890
+ "reads_keep1": 2.953857421875,
2891
+ "reads_keep3": 2.9010009765625
2892
+ }
2893
+ ],
2894
+ "step_s": 0.8337,
2895
+ "grad_norm": 0.20284,
2896
+ "grad_norm_mean": 0.19793,
2897
+ "clip_frac": 0.0,
2898
+ "pairs_per_step": 786432.0,
2899
+ "max_mem_alloc_gb": [
2900
+ 21.7,
2901
+ 21.72,
2902
+ 21.71,
2903
+ 21.71,
2904
+ 21.7,
2905
+ 21.7,
2906
+ 21.7,
2907
+ 21.72
2908
+ ],
2909
+ "max_mem_reserved_gb": [
2910
+ 22.04,
2911
+ 22.06,
2912
+ 22.06,
2913
+ 22.06,
2914
+ 22.06,
2915
+ 22.04,
2916
+ 22.05,
2917
+ 22.09
2918
+ ]
2919
+ },
2920
+ {
2921
+ "step": 5600,
2922
+ "loss": 2.9599080085754395,
2923
+ "lr": 0.00015584225419691021,
2924
+ "elapsed_s": 4253.901607513428,
2925
+ "util": [
2926
+ 0.98,
2927
+ 0.9961
2928
+ ],
2929
+ "drop": [
2930
+ 0.0,
2931
+ 0.0
2932
+ ],
2933
+ "traffic": [
2934
+ {
2935
+ "k": 3,
2936
+ "reads_keep1": 2.9940185546875,
2937
+ "reads_keep3": 2.9664306640625
2938
+ },
2939
+ {
2940
+ "k": 3,
2941
+ "reads_keep1": 2.97705078125,
2942
+ "reads_keep3": 2.9482421875
2943
+ }
2944
+ ],
2945
+ "step_s": 0.8245,
2946
+ "grad_norm": 0.21032,
2947
+ "grad_norm_mean": 0.20075,
2948
+ "clip_frac": 0.0,
2949
+ "pairs_per_step": 786432.0,
2950
+ "max_mem_alloc_gb": [
2951
+ 21.7,
2952
+ 21.72,
2953
+ 21.71,
2954
+ 21.71,
2955
+ 21.7,
2956
+ 21.7,
2957
+ 21.7,
2958
+ 21.72
2959
+ ],
2960
+ "max_mem_reserved_gb": [
2961
+ 22.04,
2962
+ 22.06,
2963
+ 22.06,
2964
+ 22.06,
2965
+ 22.06,
2966
+ 22.04,
2967
+ 22.05,
2968
+ 22.09
2969
+ ]
2970
+ },
2971
+ {
2972
+ "step": 5700,
2973
+ "loss": 3.054381847381592,
2974
+ "lr": 0.00014716086675636405,
2975
+ "elapsed_s": 4333.841708660126,
2976
+ "util": [
2977
+ 0.9834,
2978
+ 0.999
2979
+ ],
2980
+ "drop": [
2981
+ 0.0,
2982
+ 0.0
2983
+ ],
2984
+ "traffic": [
2985
+ {
2986
+ "k": 3,
2987
+ "reads_keep1": 2.990966796875,
2988
+ "reads_keep3": 2.9639892578125
2989
+ },
2990
+ {
2991
+ "k": 3,
2992
+ "reads_keep1": 2.9715576171875,
2993
+ "reads_keep3": 2.942626953125
2994
+ }
2995
+ ],
2996
+ "step_s": 0.7994,
2997
+ "grad_norm": 0.19443,
2998
+ "grad_norm_mean": 0.1994,
2999
+ "clip_frac": 0.0,
3000
+ "pairs_per_step": 786432.0,
3001
+ "max_mem_alloc_gb": [
3002
+ 21.7,
3003
+ 21.72,
3004
+ 21.71,
3005
+ 21.71,
3006
+ 21.7,
3007
+ 21.7,
3008
+ 21.7,
3009
+ 21.72
3010
+ ],
3011
+ "max_mem_reserved_gb": [
3012
+ 22.04,
3013
+ 22.06,
3014
+ 22.06,
3015
+ 22.06,
3016
+ 22.06,
3017
+ 22.04,
3018
+ 22.05,
3019
+ 22.09
3020
+ ]
3021
+ },
3022
+ {
3023
+ "step": 5800,
3024
+ "loss": 2.800983428955078,
3025
+ "lr": 0.0001388153815992021,
3026
+ "elapsed_s": 4414.956063985825,
3027
+ "util": [
3028
+ 0.9829,
3029
+ 0.998
3030
+ ],
3031
+ "drop": [
3032
+ 0.0,
3033
+ 0.0
3034
+ ],
3035
+ "traffic": [
3036
+ {
3037
+ "k": 3,
3038
+ "reads_keep1": 2.9940185546875,
3039
+ "reads_keep3": 2.9686279296875
3040
+ },
3041
+ {
3042
+ "k": 3,
3043
+ "reads_keep1": 2.977294921875,
3044
+ "reads_keep3": 2.9537353515625
3045
+ }
3046
+ ],
3047
+ "step_s": 0.8112,
3048
+ "grad_norm": 0.20158,
3049
+ "grad_norm_mean": 0.19817,
3050
+ "clip_frac": 0.0,
3051
+ "pairs_per_step": 786432.0,
3052
+ "max_mem_alloc_gb": [
3053
+ 21.7,
3054
+ 21.72,
3055
+ 21.71,
3056
+ 21.71,
3057
+ 21.7,
3058
+ 21.7,
3059
+ 21.7,
3060
+ 21.72
3061
+ ],
3062
+ "max_mem_reserved_gb": [
3063
+ 22.04,
3064
+ 22.06,
3065
+ 22.06,
3066
+ 22.06,
3067
+ 22.06,
3068
+ 22.04,
3069
+ 22.05,
3070
+ 22.09
3071
+ ]
3072
+ },
3073
+ {
3074
+ "step": 5900,
3075
+ "loss": 3.066985607147217,
3076
+ "lr": 0.00013082113060471372,
3077
+ "elapsed_s": 4496.873050928116,
3078
+ "util": [
3079
+ 0.9834,
3080
+ 0.9961
3081
+ ],
3082
+ "drop": [
3083
+ 0.0,
3084
+ 0.0
3085
+ ],
3086
+ "traffic": [
3087
+ {
3088
+ "k": 3,
3089
+ "reads_keep1": 2.9921875,
3090
+ "reads_keep3": 2.9593505859375
3091
+ },
3092
+ {
3093
+ "k": 3,
3094
+ "reads_keep1": 2.976318359375,
3095
+ "reads_keep3": 2.9468994140625
3096
+ }
3097
+ ],
3098
+ "step_s": 0.8192,
3099
+ "grad_norm": 0.2002,
3100
+ "grad_norm_mean": 0.1975,
3101
+ "clip_frac": 0.0,
3102
+ "pairs_per_step": 786432.0,
3103
+ "max_mem_alloc_gb": [
3104
+ 21.7,
3105
+ 21.72,
3106
+ 21.71,
3107
+ 21.71,
3108
+ 21.7,
3109
+ 21.7,
3110
+ 21.7,
3111
+ 21.72
3112
+ ],
3113
+ "max_mem_reserved_gb": [
3114
+ 22.04,
3115
+ 22.06,
3116
+ 22.06,
3117
+ 22.06,
3118
+ 22.06,
3119
+ 22.04,
3120
+ 22.05,
3121
+ 22.09
3122
+ ]
3123
+ },
3124
+ {
3125
+ "step": 6000,
3126
+ "loss": 2.691514492034912,
3127
+ "lr": 0.0001231928003835568,
3128
+ "elapsed_s": 4576.404948949814,
3129
+ "util": [
3130
+ 0.9858,
3131
+ 0.9985
3132
+ ],
3133
+ "drop": [
3134
+ 0.0,
3135
+ 0.0
3136
+ ],
3137
+ "traffic": [
3138
+ {
3139
+ "k": 3,
3140
+ "reads_keep1": 2.991455078125,
3141
+ "reads_keep3": 2.960693359375
3142
+ },
3143
+ {
3144
+ "k": 3,
3145
+ "reads_keep1": 2.9720458984375,
3146
+ "reads_keep3": 2.9393310546875
3147
+ }
3148
+ ],
3149
+ "step_s": 0.7953,
3150
+ "grad_norm": 0.20242,
3151
+ "grad_norm_mean": 0.2039,
3152
+ "clip_frac": 0.0,
3153
+ "pairs_per_step": 786432.0,
3154
+ "max_mem_alloc_gb": [
3155
+ 21.7,
3156
+ 21.72,
3157
+ 21.71,
3158
+ 21.71,
3159
+ 21.7,
3160
+ 21.7,
3161
+ 21.7,
3162
+ 21.72
3163
+ ],
3164
+ "max_mem_reserved_gb": [
3165
+ 22.04,
3166
+ 22.06,
3167
+ 22.06,
3168
+ 22.06,
3169
+ 22.06,
3170
+ 22.04,
3171
+ 22.05,
3172
+ 22.09
3173
+ ]
3174
+ },
3175
+ {
3176
+ "step": 6100,
3177
+ "loss": 2.883620262145996,
3178
+ "lr": 0.00011594440529630196,
3179
+ "elapsed_s": 4657.44682765007,
3180
+ "util": [
3181
+ 0.9902,
3182
+ 0.9985
3183
+ ],
3184
+ "drop": [
3185
+ 0.0,
3186
+ 0.0
3187
+ ],
3188
+ "traffic": [
3189
+ {
3190
+ "k": 3,
3191
+ "reads_keep1": 2.9918212890625,
3192
+ "reads_keep3": 2.9710693359375
3193
+ },
3194
+ {
3195
+ "k": 3,
3196
+ "reads_keep1": 2.976318359375,
3197
+ "reads_keep3": 2.947509765625
3198
+ }
3199
+ ],
3200
+ "step_s": 0.8104,
3201
+ "grad_norm": 0.21226,
3202
+ "grad_norm_mean": 0.19976,
3203
+ "clip_frac": 0.0,
3204
+ "pairs_per_step": 786432.0,
3205
+ "max_mem_alloc_gb": [
3206
+ 21.7,
3207
+ 21.72,
3208
+ 21.71,
3209
+ 21.71,
3210
+ 21.7,
3211
+ 21.7,
3212
+ 21.7,
3213
+ 21.72
3214
+ ],
3215
+ "max_mem_reserved_gb": [
3216
+ 22.04,
3217
+ 22.06,
3218
+ 22.06,
3219
+ 22.06,
3220
+ 22.06,
3221
+ 22.04,
3222
+ 22.05,
3223
+ 22.09
3224
+ ]
3225
+ },
3226
+ {
3227
+ "step": 6200,
3228
+ "loss": 2.6363463401794434,
3229
+ "lr": 0.00010908926170699825,
3230
+ "elapsed_s": 4737.542729854584,
3231
+ "util": [
3232
+ 0.9873,
3233
+ 0.999
3234
+ ],
3235
+ "drop": [
3236
+ 0.0,
3237
+ 0.0
3238
+ ],
3239
+ "traffic": [
3240
+ {
3241
+ "k": 3,
3242
+ "reads_keep1": 2.9923095703125,
3243
+ "reads_keep3": 2.96728515625
3244
+ },
3245
+ {
3246
+ "k": 3,
3247
+ "reads_keep1": 2.972900390625,
3248
+ "reads_keep3": 2.946533203125
3249
+ }
3250
+ ],
3251
+ "step_s": 0.801,
3252
+ "grad_norm": 0.18963,
3253
+ "grad_norm_mean": 0.2015,
3254
+ "clip_frac": 0.0,
3255
+ "pairs_per_step": 786432.0,
3256
+ "max_mem_alloc_gb": [
3257
+ 21.7,
3258
+ 21.72,
3259
+ 21.71,
3260
+ 21.71,
3261
+ 21.7,
3262
+ 21.7,
3263
+ 21.7,
3264
+ 21.72
3265
+ ],
3266
+ "max_mem_reserved_gb": [
3267
+ 22.04,
3268
+ 22.06,
3269
+ 22.06,
3270
+ 22.06,
3271
+ 22.06,
3272
+ 22.04,
3273
+ 22.05,
3274
+ 22.09
3275
+ ]
3276
+ },
3277
+ {
3278
+ "step": 6300,
3279
+ "loss": 2.6801962852478027,
3280
+ "lr": 0.00010263996351906082,
3281
+ "elapsed_s": 4817.433548927307,
3282
+ "util": [
3283
+ 0.9878,
3284
+ 1.0
3285
+ ],
3286
+ "drop": [
3287
+ 0.0,
3288
+ 0.0
3289
+ ],
3290
+ "traffic": [
3291
+ {
3292
+ "k": 3,
3293
+ "reads_keep1": 2.9910888671875,
3294
+ "reads_keep3": 2.963134765625
3295
+ },
3296
+ {
3297
+ "k": 3,
3298
+ "reads_keep1": 2.97265625,
3299
+ "reads_keep3": 2.945068359375
3300
+ }
3301
+ ],
3302
+ "step_s": 0.7989,
3303
+ "grad_norm": 0.19201,
3304
+ "grad_norm_mean": 0.19929,
3305
+ "clip_frac": 0.0,
3306
+ "pairs_per_step": 786432.0,
3307
+ "max_mem_alloc_gb": [
3308
+ 21.7,
3309
+ 21.72,
3310
+ 21.71,
3311
+ 21.71,
3312
+ 21.7,
3313
+ 21.7,
3314
+ 21.7,
3315
+ 21.72
3316
+ ],
3317
+ "max_mem_reserved_gb": [
3318
+ 22.04,
3319
+ 22.06,
3320
+ 22.06,
3321
+ 22.06,
3322
+ 22.06,
3323
+ 22.04,
3324
+ 22.05,
3325
+ 22.09
3326
+ ]
3327
+ },
3328
+ {
3329
+ "step": 6400,
3330
+ "loss": 2.8494482040405273,
3331
+ "lr": 9.660835903842457e-05,
3332
+ "elapsed_s": 4899.8473970890045,
3333
+ "util": [
3334
+ 0.9897,
3335
+ 0.9995
3336
+ ],
3337
+ "drop": [
3338
+ 0.0,
3339
+ 0.0
3340
+ ],
3341
+ "traffic": [
3342
+ {
3343
+ "k": 3,
3344
+ "reads_keep1": 2.9908447265625,
3345
+ "reads_keep3": 2.95361328125
3346
+ },
3347
+ {
3348
+ "k": 3,
3349
+ "reads_keep1": 2.9683837890625,
3350
+ "reads_keep3": 2.929931640625
3351
+ }
3352
+ ],
3353
+ "step_s": 0.8242,
3354
+ "grad_norm": 0.20683,
3355
+ "grad_norm_mean": 0.19865,
3356
+ "clip_frac": 0.0,
3357
+ "pairs_per_step": 786432.0,
3358
+ "max_mem_alloc_gb": [
3359
+ 21.7,
3360
+ 21.72,
3361
+ 21.71,
3362
+ 21.71,
3363
+ 21.7,
3364
+ 21.7,
3365
+ 21.7,
3366
+ 21.72
3367
+ ],
3368
+ "max_mem_reserved_gb": [
3369
+ 22.04,
3370
+ 22.06,
3371
+ 22.06,
3372
+ 22.06,
3373
+ 22.06,
3374
+ 22.04,
3375
+ 22.05,
3376
+ 22.09
3377
+ ]
3378
+ },
3379
+ {
3380
+ "step": 6500,
3381
+ "loss": 2.900207996368408,
3382
+ "lr": 9.100552920646992e-05,
3383
+ "elapsed_s": 4980.584800720215,
3384
+ "util": [
3385
+ 0.9907,
3386
+ 0.9985
3387
+ ],
3388
+ "drop": [
3389
+ 0.0,
3390
+ 0.0
3391
+ ],
3392
+ "traffic": [
3393
+ {
3394
+ "k": 3,
3395
+ "reads_keep1": 2.993408203125,
3396
+ "reads_keep3": 2.9638671875
3397
+ },
3398
+ {
3399
+ "k": 3,
3400
+ "reads_keep1": 2.9718017578125,
3401
+ "reads_keep3": 2.943115234375
3402
+ }
3403
+ ],
3404
+ "step_s": 0.8074,
3405
+ "grad_norm": 0.19677,
3406
+ "grad_norm_mean": 0.19913,
3407
+ "clip_frac": 0.0,
3408
+ "pairs_per_step": 786432.0,
3409
+ "max_mem_alloc_gb": [
3410
+ 21.7,
3411
+ 21.72,
3412
+ 21.71,
3413
+ 21.71,
3414
+ 21.7,
3415
+ 21.7,
3416
+ 21.7,
3417
+ 21.72
3418
+ ],
3419
+ "max_mem_reserved_gb": [
3420
+ 22.04,
3421
+ 22.06,
3422
+ 22.06,
3423
+ 22.06,
3424
+ 22.06,
3425
+ 22.04,
3426
+ 22.05,
3427
+ 22.09
3428
+ ]
3429
+ },
3430
+ {
3431
+ "step": 6600,
3432
+ "loss": 2.6298065185546875,
3433
+ "lr": 8.584176724270953e-05,
3434
+ "elapsed_s": 5061.568926811218,
3435
+ "util": [
3436
+ 0.9941,
3437
+ 0.999
3438
+ ],
3439
+ "drop": [
3440
+ 0.0,
3441
+ 0.0
3442
+ ],
3443
+ "traffic": [
3444
+ {
3445
+ "k": 3,
3446
+ "reads_keep1": 2.989013671875,
3447
+ "reads_keep3": 2.95654296875
3448
+ },
3449
+ {
3450
+ "k": 3,
3451
+ "reads_keep1": 2.9732666015625,
3452
+ "reads_keep3": 2.94384765625
3453
+ }
3454
+ ],
3455
+ "step_s": 0.8098,
3456
+ "grad_norm": 0.19845,
3457
+ "grad_norm_mean": 0.19698,
3458
+ "clip_frac": 0.0,
3459
+ "pairs_per_step": 786432.0,
3460
+ "max_mem_alloc_gb": [
3461
+ 21.7,
3462
+ 21.72,
3463
+ 21.71,
3464
+ 21.71,
3465
+ 21.7,
3466
+ 21.7,
3467
+ 21.7,
3468
+ 21.72
3469
+ ],
3470
+ "max_mem_reserved_gb": [
3471
+ 22.04,
3472
+ 22.06,
3473
+ 22.06,
3474
+ 22.06,
3475
+ 22.06,
3476
+ 22.04,
3477
+ 22.05,
3478
+ 22.09
3479
+ ]
3480
+ },
3481
+ {
3482
+ "step": 6700,
3483
+ "loss": 2.5884947776794434,
3484
+ "lr": 8.112655973463532e-05,
3485
+ "elapsed_s": 5140.840366840363,
3486
+ "util": [
3487
+ 0.9907,
3488
+ 1.0
3489
+ ],
3490
+ "drop": [
3491
+ 0.0,
3492
+ 0.0
3493
+ ],
3494
+ "traffic": [
3495
+ {
3496
+ "k": 3,
3497
+ "reads_keep1": 2.98779296875,
3498
+ "reads_keep3": 2.9552001953125
3499
+ },
3500
+ {
3501
+ "k": 3,
3502
+ "reads_keep1": 2.9674072265625,
3503
+ "reads_keep3": 2.9361572265625
3504
+ }
3505
+ ],
3506
+ "step_s": 0.7927,
3507
+ "grad_norm": 0.19382,
3508
+ "grad_norm_mean": 0.19712,
3509
+ "clip_frac": 0.0,
3510
+ "pairs_per_step": 786432.0,
3511
+ "max_mem_alloc_gb": [
3512
+ 21.7,
3513
+ 21.72,
3514
+ 21.71,
3515
+ 21.71,
3516
+ 21.7,
3517
+ 21.7,
3518
+ 21.7,
3519
+ 21.72
3520
+ ],
3521
+ "max_mem_reserved_gb": [
3522
+ 22.04,
3523
+ 22.06,
3524
+ 22.06,
3525
+ 22.06,
3526
+ 22.06,
3527
+ 22.04,
3528
+ 22.05,
3529
+ 22.09
3530
+ ]
3531
+ },
3532
+ {
3533
+ "step": 6800,
3534
+ "loss": 2.826439380645752,
3535
+ "lr": 7.686856920946722e-05,
3536
+ "elapsed_s": 5222.547169446945,
3537
+ "util": [
3538
+ 0.9932,
3539
+ 0.999
3540
+ ],
3541
+ "drop": [
3542
+ 0.0,
3543
+ 0.0
3544
+ ],
3545
+ "traffic": [
3546
+ {
3547
+ "k": 3,
3548
+ "reads_keep1": 2.9931640625,
3549
+ "reads_keep3": 2.9654541015625
3550
+ },
3551
+ {
3552
+ "k": 3,
3553
+ "reads_keep1": 2.9737548828125,
3554
+ "reads_keep3": 2.9482421875
3555
+ }
3556
+ ],
3557
+ "step_s": 0.817,
3558
+ "grad_norm": 0.20056,
3559
+ "grad_norm_mean": 0.19746,
3560
+ "clip_frac": 0.0,
3561
+ "pairs_per_step": 786432.0,
3562
+ "max_mem_alloc_gb": [
3563
+ 21.7,
3564
+ 21.72,
3565
+ 21.71,
3566
+ 21.71,
3567
+ 21.7,
3568
+ 21.7,
3569
+ 21.7,
3570
+ 21.72
3571
+ ],
3572
+ "max_mem_reserved_gb": [
3573
+ 22.04,
3574
+ 22.06,
3575
+ 22.06,
3576
+ 22.06,
3577
+ 22.06,
3578
+ 22.04,
3579
+ 22.05,
3580
+ 22.09
3581
+ ]
3582
+ },
3583
+ {
3584
+ "step": 6900,
3585
+ "loss": 2.579041004180908,
3586
+ "lr": 7.307561821982112e-05,
3587
+ "elapsed_s": 5312.427621841431,
3588
+ "util": [
3589
+ 0.9907,
3590
+ 0.9995
3591
+ ],
3592
+ "drop": [
3593
+ 0.0,
3594
+ 0.0
3595
+ ],
3596
+ "traffic": [
3597
+ {
3598
+ "k": 3,
3599
+ "reads_keep1": 2.9896240234375,
3600
+ "reads_keep3": 2.9691162109375
3601
+ },
3602
+ {
3603
+ "k": 3,
3604
+ "reads_keep1": 2.971435546875,
3605
+ "reads_keep3": 2.9505615234375
3606
+ }
3607
+ ],
3608
+ "step_s": 0.8988,
3609
+ "grad_norm": 0.19895,
3610
+ "grad_norm_mean": 0.19855,
3611
+ "clip_frac": 0.0,
3612
+ "pairs_per_step": 786432.0,
3613
+ "max_mem_alloc_gb": [
3614
+ 21.7,
3615
+ 21.72,
3616
+ 21.71,
3617
+ 21.71,
3618
+ 21.7,
3619
+ 21.7,
3620
+ 21.7,
3621
+ 21.72
3622
+ ],
3623
+ "max_mem_reserved_gb": [
3624
+ 22.04,
3625
+ 22.06,
3626
+ 22.06,
3627
+ 22.06,
3628
+ 22.06,
3629
+ 22.04,
3630
+ 22.05,
3631
+ 22.09
3632
+ ]
3633
+ },
3634
+ {
3635
+ "step": 7000,
3636
+ "loss": 2.8963675498962402,
3637
+ "lr": 6.975467497253317e-05,
3638
+ "elapsed_s": 5350.748041152954,
3639
+ "util": [
3640
+ 0.9937,
3641
+ 0.9985
3642
+ ],
3643
+ "drop": [
3644
+ 0.0,
3645
+ 0.0
3646
+ ],
3647
+ "traffic": [
3648
+ {
3649
+ "k": 3,
3650
+ "reads_keep1": 2.9918212890625,
3651
+ "reads_keep3": 2.967529296875
3652
+ },
3653
+ {
3654
+ "k": 3,
3655
+ "reads_keep1": 2.970458984375,
3656
+ "reads_keep3": 2.946533203125
3657
+ }
3658
+ ],
3659
+ "step_s": 0.3835,
3660
+ "grad_norm": 0.20476,
3661
+ "grad_norm_mean": 0.19756,
3662
+ "clip_frac": 0.0,
3663
+ "pairs_per_step": 786432.0,
3664
+ "max_mem_alloc_gb": [
3665
+ 21.7,
3666
+ 21.72,
3667
+ 21.71,
3668
+ 21.71,
3669
+ 21.7,
3670
+ 21.7,
3671
+ 21.7,
3672
+ 21.72
3673
+ ],
3674
+ "max_mem_reserved_gb": [
3675
+ 22.04,
3676
+ 22.06,
3677
+ 22.06,
3678
+ 22.06,
3679
+ 22.06,
3680
+ 22.04,
3681
+ 22.05,
3682
+ 22.09
3683
+ ]
3684
+ },
3685
+ {
3686
+ "step": 7100,
3687
+ "loss": 2.4268391132354736,
3688
+ "lr": 6.691184052704267e-05,
3689
+ "elapsed_s": 5378.028213739395,
3690
+ "util": [
3691
+ 0.9922,
3692
+ 0.9976
3693
+ ],
3694
+ "drop": [
3695
+ 0.0,
3696
+ 0.0
3697
+ ],
3698
+ "traffic": [
3699
+ {
3700
+ "k": 3,
3701
+ "reads_keep1": 2.9952392578125,
3702
+ "reads_keep3": 2.9696044921875
3703
+ },
3704
+ {
3705
+ "k": 3,
3706
+ "reads_keep1": 2.9754638671875,
3707
+ "reads_keep3": 2.939453125
3708
+ }
3709
+ ],
3710
+ "step_s": 0.2728,
3711
+ "grad_norm": 0.19741,
3712
+ "grad_norm_mean": 0.19865,
3713
+ "clip_frac": 0.0,
3714
+ "pairs_per_step": 786432.0,
3715
+ "max_mem_alloc_gb": [
3716
+ 21.7,
3717
+ 21.72,
3718
+ 21.71,
3719
+ 21.71,
3720
+ 21.7,
3721
+ 21.7,
3722
+ 21.7,
3723
+ 21.72
3724
+ ],
3725
+ "max_mem_reserved_gb": [
3726
+ 22.04,
3727
+ 22.06,
3728
+ 22.06,
3729
+ 22.06,
3730
+ 22.06,
3731
+ 22.04,
3732
+ 22.05,
3733
+ 22.09
3734
+ ]
3735
+ },
3736
+ {
3737
+ "step": 7200,
3738
+ "loss": 3.0772056579589844,
3739
+ "lr": 6.455233758685185e-05,
3740
+ "elapsed_s": 5405.304255485535,
3741
+ "util": [
3742
+ 0.9912,
3743
+ 0.9985
3744
+ ],
3745
+ "drop": [
3746
+ 0.0,
3747
+ 0.0
3748
+ ],
3749
+ "traffic": [
3750
+ {
3751
+ "k": 3,
3752
+ "reads_keep1": 2.9912109375,
3753
+ "reads_keep3": 2.9609375
3754
+ },
3755
+ {
3756
+ "k": 3,
3757
+ "reads_keep1": 2.974365234375,
3758
+ "reads_keep3": 2.9376220703125
3759
+ }
3760
+ ],
3761
+ "step_s": 0.2728,
3762
+ "grad_norm": 0.20115,
3763
+ "grad_norm_mean": 0.19978,
3764
+ "clip_frac": 0.0,
3765
+ "pairs_per_step": 786432.0,
3766
+ "max_mem_alloc_gb": [
3767
+ 21.7,
3768
+ 21.72,
3769
+ 21.71,
3770
+ 21.71,
3771
+ 21.7,
3772
+ 21.7,
3773
+ 21.7,
3774
+ 21.72
3775
+ ],
3776
+ "max_mem_reserved_gb": [
3777
+ 22.04,
3778
+ 22.06,
3779
+ 22.06,
3780
+ 22.06,
3781
+ 22.06,
3782
+ 22.04,
3783
+ 22.05,
3784
+ 22.09
3785
+ ]
3786
+ },
3787
+ {
3788
+ "step": 7300,
3789
+ "loss": 2.7156317234039307,
3790
+ "lr": 6.268050090465387e-05,
3791
+ "elapsed_s": 5432.576008319855,
3792
+ "util": [
3793
+ 0.9946,
3794
+ 1.0
3795
+ ],
3796
+ "drop": [
3797
+ 0.0,
3798
+ 0.0
3799
+ ],
3800
+ "traffic": [
3801
+ {
3802
+ "k": 3,
3803
+ "reads_keep1": 2.990478515625,
3804
+ "reads_keep3": 2.9658203125
3805
+ },
3806
+ {
3807
+ "k": 3,
3808
+ "reads_keep1": 2.9688720703125,
3809
+ "reads_keep3": 2.941162109375
3810
+ }
3811
+ ],
3812
+ "step_s": 0.2727,
3813
+ "grad_norm": 0.20153,
3814
+ "grad_norm_mean": 0.19968,
3815
+ "clip_frac": 0.0,
3816
+ "pairs_per_step": 786432.0,
3817
+ "max_mem_alloc_gb": [
3818
+ 21.7,
3819
+ 21.72,
3820
+ 21.71,
3821
+ 21.71,
3822
+ 21.7,
3823
+ 21.7,
3824
+ 21.7,
3825
+ 21.72
3826
+ ],
3827
+ "max_mem_reserved_gb": [
3828
+ 22.04,
3829
+ 22.06,
3830
+ 22.06,
3831
+ 22.06,
3832
+ 22.06,
3833
+ 22.04,
3834
+ 22.05,
3835
+ 22.09
3836
+ ]
3837
+ },
3838
+ {
3839
+ "step": 7400,
3840
+ "loss": 2.7013535499572754,
3841
+ "lr": 6.12997693187572e-05,
3842
+ "elapsed_s": 5459.862614631653,
3843
+ "util": [
3844
+ 0.9956,
3845
+ 0.9995
3846
+ ],
3847
+ "drop": [
3848
+ 0.0,
3849
+ 0.0
3850
+ ],
3851
+ "traffic": [
3852
+ {
3853
+ "k": 3,
3854
+ "reads_keep1": 2.9925537109375,
3855
+ "reads_keep3": 2.9715576171875
3856
+ },
3857
+ {
3858
+ "k": 3,
3859
+ "reads_keep1": 2.97021484375,
3860
+ "reads_keep3": 2.9449462890625
3861
+ }
3862
+ ],
3863
+ "step_s": 0.2729,
3864
+ "grad_norm": 0.20484,
3865
+ "grad_norm_mean": 0.20115,
3866
+ "clip_frac": 0.0,
3867
+ "pairs_per_step": 786432.0,
3868
+ "max_mem_alloc_gb": [
3869
+ 21.7,
3870
+ 21.72,
3871
+ 21.71,
3872
+ 21.71,
3873
+ 21.7,
3874
+ 21.7,
3875
+ 21.7,
3876
+ 21.72
3877
+ ],
3878
+ "max_mem_reserved_gb": [
3879
+ 22.04,
3880
+ 22.06,
3881
+ 22.06,
3882
+ 22.06,
3883
+ 22.06,
3884
+ 22.04,
3885
+ 22.05,
3886
+ 22.09
3887
+ ]
3888
+ },
3889
+ {
3890
+ "step": 7500,
3891
+ "loss": 2.566859245300293,
3892
+ "lr": 6.0412679435435414e-05,
3893
+ "elapsed_s": 5487.134707212448,
3894
+ "util": [
3895
+ 0.9941,
3896
+ 0.9985
3897
+ ],
3898
+ "drop": [
3899
+ 0.0,
3900
+ 0.0
3901
+ ],
3902
+ "traffic": [
3903
+ {
3904
+ "k": 3,
3905
+ "reads_keep1": 2.99365234375,
3906
+ "reads_keep3": 2.9697265625
3907
+ },
3908
+ {
3909
+ "k": 3,
3910
+ "reads_keep1": 2.9718017578125,
3911
+ "reads_keep3": 2.9490966796875
3912
+ }
3913
+ ],
3914
+ "step_s": 0.2727,
3915
+ "grad_norm": 0.19749,
3916
+ "grad_norm_mean": 0.20082,
3917
+ "clip_frac": 0.0,
3918
+ "pairs_per_step": 786432.0,
3919
+ "max_mem_alloc_gb": [
3920
+ 21.7,
3921
+ 21.72,
3922
+ 21.71,
3923
+ 21.71,
3924
+ 21.7,
3925
+ 21.7,
3926
+ 21.7,
3927
+ 21.72
3928
+ ],
3929
+ "max_mem_reserved_gb": [
3930
+ 22.04,
3931
+ 22.06,
3932
+ 22.06,
3933
+ 22.06,
3934
+ 22.06,
3935
+ 22.04,
3936
+ 22.05,
3937
+ 22.09
3938
+ ]
3939
+ },
3940
+ {
3941
+ "step": 7600,
3942
+ "loss": 2.9380922317504883,
3943
+ "lr": 6.0020860968809794e-05,
3944
+ "elapsed_s": 5514.400843381882,
3945
+ "util": [
3946
+ 0.9932,
3947
+ 0.9985
3948
+ ],
3949
+ "drop": [
3950
+ 0.0,
3951
+ 0.0
3952
+ ],
3953
+ "traffic": [
3954
+ {
3955
+ "k": 3,
3956
+ "reads_keep1": 2.993408203125,
3957
+ "reads_keep3": 2.961669921875
3958
+ },
3959
+ {
3960
+ "k": 3,
3961
+ "reads_keep1": 2.97216796875,
3962
+ "reads_keep3": 2.936767578125
3963
+ }
3964
+ ],
3965
+ "step_s": 0.2727,
3966
+ "grad_norm": 0.19654,
3967
+ "grad_norm_mean": 0.20245,
3968
+ "clip_frac": 0.0,
3969
+ "pairs_per_step": 786432.0,
3970
+ "max_mem_alloc_gb": [
3971
+ 21.7,
3972
+ 21.72,
3973
+ 21.71,
3974
+ 21.71,
3975
+ 21.7,
3976
+ 21.7,
3977
+ 21.7,
3978
+ 21.72
3979
+ ],
3980
+ "max_mem_reserved_gb": [
3981
+ 22.04,
3982
+ 22.06,
3983
+ 22.06,
3984
+ 22.06,
3985
+ 22.06,
3986
+ 22.04,
3987
+ 22.05,
3988
+ 22.09
3989
+ ]
3990
+ }
3991
+ ],
3992
+ "wall_s": 5571.792639493942,
3993
+ "resumed_from_step": 0,
3994
+ "resume_dtype": null,
3995
+ "expert_wd": 0.1,
3996
+ "expert_lr_mult": 1.0,
3997
+ "lr_mult": 1.0,
3998
+ "tokens_freeze": "3cb3d04a50816f7a",
3999
+ "d_ff_e": 853,
4000
+ "expert_width": 213,
4001
+ "experts_per_layer": 2048,
4002
+ "k_active": 3,
4003
+ "n_shared": 1,
4004
+ "units_per_layer": 2048,
4005
+ "unit_params": 736128,
4006
+ "ep": 8,
4007
+ "argv": [
4008
+ "src/launch_tokens.py",
4009
+ "--out",
4010
+ "/tmp/jalenl/dev/gate_b_tier",
4011
+ "--arm",
4012
+ "8",
4013
+ "--mult",
4014
+ "1",
4015
+ "--root",
4016
+ "/tmp/jalenl/v4",
4017
+ "--store-norm",
4018
+ "--gran",
4019
+ "4",
4020
+ "--shared",
4021
+ "1",
4022
+ "--router",
4023
+ "learned",
4024
+ "--cap-factor",
4025
+ "0",
4026
+ "--ep",
4027
+ "--micro-batch",
4028
+ "4",
4029
+ "--accum",
4030
+ "2",
4031
+ "--tier-gather"
4032
+ ],
4033
+ "seq_ids": {
4034
+ "step0": [
4035
+ 56082,
4036
+ 419616
4037
+ ],
4038
+ "step1": [
4039
+ 287328,
4040
+ 497789
4041
+ ],
4042
+ "sha256_steps01": "f55a693087f3af64"
4043
+ }
4044
+ }