suchirsalhan commited on
Commit
4e1af4d
·
verified ·
1 Parent(s): 56f9904

run records: final.json, log, probes

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
config.yaml ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model: {architecture: hnet, d_enc: 256, d_main: 512, n_enc: 2, n_dec: 2, n_main: 6,
2
+ n_heads_enc: 4, n_heads_main: 8, seq_len: 1024, dropout: 0.0, target_ratio: 6.0,
3
+ max_chunk_mult: 3.0, dechunk: ste, router_noise: 0.0, rope_base: 10000.0}
4
+ memory:
5
+ ali:
6
+ budget_B: 65536
7
+ staging_capacity: 262144
8
+ policy: utility
9
+ candidate_types: [surface, split, merge, skel]
10
+ write_rule: mean
11
+ eta_min: 0.0
12
+ memory_scope: persistent
13
+ consolidate_every: 50
14
+ max_span: 64
15
+ gate: vector
16
+ retrieval: true
17
+ read_dropout: 0.1
18
+ gate_hidden: 0
19
+ type_dim: 16
20
+ gate_bias_init: -2.0
21
+ probe_seqs: 2
22
+ probe_reps: 8
23
+ horizon: 32
24
+ predictor: mlp
25
+ lambda_util: 0.1
26
+ enabled: true
27
+ inventory_kwargs: {}
28
+ probe_mix: null
29
+ util_batch_slots: 1024
30
+ pkm: {n_keys: 256, topk: 8, heads: 1, d_key: null, value_lr_mult: 1.0}
31
+ objective: {name: parity, schedule: B, alpha: 0.03, alpha_parity_cap: 0.06, global_anchor: 0.0,
32
+ premium_clip: 4.0, premium_kind: flores_per_sent, tau1: 0.4, tau2: 0.7, freeze_scope: router,
33
+ cpc_decay: 0.9, genreg_target_entropy: null, genreg_eta: 0.05}
34
+ data: {corpus: corpus, langs: null, mixture: null, doc_filters: null, byte_classes: scratch/data/byte_classes.json}
35
+ training:
36
+ seed: 2
37
+ max_steps: 6000
38
+ batch_size: 32
39
+ grad_accum: 1
40
+ lr: 0.0003
41
+ min_lr_frac: 0.1
42
+ warmup: 300
43
+ weight_decay: 0.1
44
+ grad_clip: 1.0
45
+ betas: [0.9, 0.95]
46
+ router_lr_mult: 1.0
47
+ log_every: 50
48
+ amp: bf16
49
+ compile: false
50
+ device: cuda
51
+ checkpointing:
52
+ run_name: parity_B_ste_s2
53
+ runs_dir: runs
54
+ ckpt_every: 200
55
+ keep: false
56
+ milestones: [1000, 3000]
57
+ resume: auto
58
+ evaluation:
59
+ eval_every: 200
60
+ eval_batches: 8
61
+ probe_sents: 128
62
+ probe_bpb_batches: 4
63
+ log_spaced_probe: true
64
+ dense_early_every: 0
65
+ dense_early_until: 0
66
+ dump_reps: true
67
+ dump_reps_thresholds: [0.3, 0.4, 0.5, 0.6, 0.7]
final.json ADDED
@@ -0,0 +1,120 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "event": "final",
3
+ "run": "parity_B_ste_s2",
4
+ "arch": "hnet",
5
+ "objective": "parity",
6
+ "condition": "B",
7
+ "step": 6000,
8
+ "params": 22516992,
9
+ "seed": 2,
10
+ "per_lang": {
11
+ "en": {
12
+ "cpc": 0.2816181118198012,
13
+ "cps": 38.2890625,
14
+ "bpb": 2.0135295581715775,
15
+ "bytes_per_chunk": 3.4952999373324976
16
+ },
17
+ "de": {
18
+ "cpc": 0.2829591389470608,
19
+ "cps": 45.390625,
20
+ "bpb": 2.1950287400328916,
21
+ "bytes_per_chunk": 3.5935981356454985
22
+ },
23
+ "vi": {
24
+ "cpc": 0.2865202061296447,
25
+ "cps": 41.265625,
26
+ "bpb": 1.8864606495338812,
27
+ "bytes_per_chunk": 4.391095328229953
28
+ },
29
+ "tr": {
30
+ "cpc": 0.33535878031265537,
31
+ "cps": 47.4296875,
32
+ "bpb": 2.1344755189754245,
33
+ "bytes_per_chunk": 3.20739988498856
34
+ },
35
+ "fi": {
36
+ "cpc": 0.28206900217957576,
37
+ "cps": 41.453125,
38
+ "bpb": 2.3628800021588425,
39
+ "bytes_per_chunk": 3.546536247471775
40
+ },
41
+ "ru": {
42
+ "cpc": 0.3092348284960422,
43
+ "cps": 45.78125,
44
+ "bpb": 1.3645900121596202,
45
+ "bytes_per_chunk": 5.797273239934983
46
+ },
47
+ "el": {
48
+ "cpc": 0.24942747161720996,
49
+ "cps": 39.9921875,
50
+ "bpb": 1.431534441739535,
51
+ "bytes_per_chunk": 7.108122398622541
52
+ },
53
+ "hi": {
54
+ "cpc": 0.32689450222882616,
55
+ "cps": 42.96875,
56
+ "bpb": 1.1660458178773176,
57
+ "bytes_per_chunk": 8.080140554202755
58
+ },
59
+ "ta": {
60
+ "cpc": 0.22733351163412677,
61
+ "cps": 33.203125,
62
+ "bpb": 0.9700471441848934,
63
+ "bytes_per_chunk": 11.362735961509287
64
+ },
65
+ "th": {
66
+ "cpc": 0.35697855144544605,
67
+ "cps": 44.859375,
68
+ "bpb": 1.0921100516892268,
69
+ "bytes_per_chunk": 8.269526813880127
70
+ },
71
+ "zh": {
72
+ "cpc": 0.5599390450389434,
73
+ "cps": 25.8359375,
74
+ "bpb": 3.106413392053783,
75
+ "bytes_per_chunk": 4.593698524515474
76
+ },
77
+ "my": {
78
+ "cpc": 0.20593202448807263,
79
+ "cps": 30.484375,
80
+ "bpb": 1.0940867342869187,
81
+ "bytes_per_chunk": 13.50631150497192
82
+ }
83
+ },
84
+ "bpb_mean": 1.734766838571992,
85
+ "gini_cpc": 0.1343910771829675,
86
+ "gini_cps": 0.08779415779415789,
87
+ "bpic": 6.286339066339067,
88
+ "mean_boundary_entropy": 0.662835955619812,
89
+ "dechunk": "ste",
90
+ "router_noise": 0.0,
91
+ "transitions": [
92
+ {
93
+ "step": 0,
94
+ "phase": 1,
95
+ "objective": "global",
96
+ "router_trainable": true,
97
+ "encoder_trainable": true,
98
+ "frozen_router_params": 0
99
+ },
100
+ {
101
+ "step": 2400,
102
+ "phase": 2,
103
+ "objective": "parity_a",
104
+ "router_trainable": true,
105
+ "encoder_trainable": true,
106
+ "frozen_router_params": 0
107
+ },
108
+ {
109
+ "step": 4200,
110
+ "phase": 3,
111
+ "objective": "none",
112
+ "router_trainable": false,
113
+ "encoder_trainable": true,
114
+ "frozen_router_params": 131584
115
+ }
116
+ ],
117
+ "probe_overhead_sec_total": 0.0,
118
+ "probe_overhead_seq_frac": 0.0,
119
+ "wall_sec": 1143.6260423660278
120
+ }
log.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
probe/000000__de.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ad41ef6dc91438d7d8f080df8b2e1edad7a8dd2b32f619cb08ada9f89fc8647f
3
+ size 303494
probe/000000__el.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9691fc89a4132a2a80f7eb490a084686f7bc8d70dad73cae53cf5c7003871f6
3
+ size 474619
probe/000000__en.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ea6484dc4ad13d9ab9144f6286d987d298b8afbce626a792d4c479f1b0d6c3f2
3
+ size 258145
probe/000000__fi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b230a5106004c0d7a82ce3d0979d5ef76890c52b698933c184f086ac7472d331
3
+ size 294036
probe/000000__hi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1c5c0aa0d1af2bb158710ec24bcb339e1800df7948b54f8f37e47663e26e10a4
3
+ size 497745
probe/000000__my.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:864d9ff9832c6516fd31aeabcfe59390153743ade4f1217de2493320182b8404
3
+ size 569220
probe/000000__ru.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7a85e7f3b7f76d19213a8fd5cae0f76c0b8ddac6ec50736809bd7c6d4d0041c2
3
+ size 438801
probe/000000__ta.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2ecc8050c19eee79044af6d86639448f75c21e0cb614af21093de69c389f1825
3
+ size 541758
probe/000000__th.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:21a9bdf5e2c4a38feac64bd84999842a030dbd90dc0c9aed5551d7c35f145ec4
3
+ size 518152
probe/000000__tr.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:46b9dc2c34e7f3baa254bcc3241ac525f802d85145fd79d0d56f6be31e7f0126
3
+ size 281520
probe/000000__vi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ade897af352283153924cb51d4dd50c78c25422cca480e9d1ea14d748956a4ed
3
+ size 348819
probe/000000__zh.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:50af0068cf7a6a58353b4f83dbeff66aa7e9c206de312faa826f363d5ef9ea4c
3
+ size 231189
probe/000001__de.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fccbf3e7aeb980c9b108504bc5503eb74138cef07edd9d2ec6cf7dd6dfc768cf
3
+ size 303220
probe/000001__el.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c5bb33a8021ce813d397e427330ce7c1c30e9ac8fa2f9d547114ebd6b08a7f10
3
+ size 474382
probe/000001__en.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:839635e92cab08693b04f120177d7d49167c65649914a54a9bb313beb96b4277
3
+ size 258126
probe/000001__fi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d25c37ee11ee8148073c0a46fcfc30a1114964229f8e74f35d1740a12aba0488
3
+ size 294231
probe/000001__hi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8f5f1f150d56ba1d8db6e3385d491ae58180f0a2a71736165503e256b545c3d1
3
+ size 497900
probe/000001__my.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5027ccf7f12397cf83d07c21b1f6ca9c5e5176098a7a8b7ce5ba7f45dd233483
3
+ size 569071
probe/000001__ru.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d5a5150cfaba866e5689e5d6f806e38f6ef40a33f2b8a76d2a08a01cee22647
3
+ size 438841
probe/000001__ta.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7ce0ddc90c05fc21c106740c3bd88ee4e3ce745f339816f858f0defb68a0ad14
3
+ size 541679
probe/000001__th.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e1c79ca65062f9fa0c20bc15a96d49aa9bb2916b879f66f0eed0813215cc040f
3
+ size 517923
probe/000001__tr.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d0598bc35a81f0bd7811625342d3c79c35e30960e844b73093ccab769ccc0372
3
+ size 281298
probe/000001__vi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2801ca69e81c1b44441092ecf091954ba36c3e6ed2000fcfc28832e0f25ec304
3
+ size 348926
probe/000001__zh.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:762852267ce383137dcf023ee2c480a8b4d15a3127c33517e0bcc81a0b0f9316
3
+ size 231391
probe/000002__de.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3765d3b821535a8c2e6e400976381ab70674954e6e1b9b3797ba1dd22cb3c538
3
+ size 303348
probe/000002__el.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aada550660683b6bc78b68eb8f7e87bb0e9c5fa7174a09f7086f0ac504bd8886
3
+ size 474448
probe/000002__en.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7e41cba94e8d1127bfd04d94776947472060e3a1507db1d627e53a49fa270e9f
3
+ size 258140
probe/000002__fi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:202b40152614d3453dc291a517a08ea49a3d2d963da8d94b64e2328f5a3607dd
3
+ size 294240
probe/000002__hi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:405ddcaec0288c6f41983e304cef1b040045bea1cfe24f3909360b79c0b36478
3
+ size 497834
probe/000002__my.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0155ab11eefb96f68854a96c90f7b0981335da488a9587e87a4ac685fe9b955a
3
+ size 569000
probe/000002__ru.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:809286ac98e5bf0039362625335d0f3dae9f4cc18dc8afd4c77b2cb047714c50
3
+ size 438897
probe/000002__ta.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7dd8368c48efaa803bd571ed2248b79fcb977605f6d6c97408512f38eecb718b
3
+ size 541842
probe/000002__th.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:78565559c75f164f3f5506b0d18179aeeefd2b89617a29e13370c6f0b00bec17
3
+ size 518250
probe/000002__tr.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9b06ff7fc29555a033c5eb11de5a47acfe8e4eaa741f333d6a25e384d8d6b447
3
+ size 281428
probe/000002__vi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:97dcdec99d7d91295706654fcba7c5b4e77cd9265c5535c4e47ffd6347872811
3
+ size 348914
probe/000002__zh.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:517392ea296173c3a0414da48138e1c878deac8ff36ac95ae13fcba9b8790e95
3
+ size 231395
probe/000004__de.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:15c10791fcf96682d641ae05344d8416e52d4d478a0afe251f562e5b2169e75d
3
+ size 303433
probe/000004__el.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3aee5b8f6ec452ed846ccd58b51531cdab8444fa1f6de94783571c461b884059
3
+ size 474750
probe/000004__en.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ac1a451d5ae2aa278560ee71e87889bc6cfe92031b899cd31ea33bd1ccb0062e
3
+ size 258053
probe/000004__fi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0676a88e31cee4b814edcc72b3584635a4870d30d151222bdea7c052d952b41
3
+ size 294449
probe/000004__hi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f3f88859a83cd6fc81bb221e70e678e56af733cfa890efca125e1e0d36e8d770
3
+ size 498190
probe/000004__my.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9f17ee726bc1d95dcb557cc59188223a414a30b49e46181b748bfbc40a4dc3f9
3
+ size 569046
probe/000004__ru.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:46ba2be3975cb0eba85e866d478f14580e0967016b9f35ba90573c4c5d16f9c5
3
+ size 439233
probe/000004__ta.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7296fa783d3f7ec2363f695565ef56503caba886ad0be6694803e6df435975da
3
+ size 542382
probe/000004__th.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6b06a3fa35a3614abce1742131c8f1950a4119eb623039aec8630c1bb4fd8e33
3
+ size 518433
probe/000004__tr.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d29c23dd0e4cdce3525157be269b874fefa450c193465998c115c798e82b1354
3
+ size 281608
probe/000004__vi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0c65338b0342102d596b7e8936c319d1b02e846b18bcf861e0f55488938c692
3
+ size 348917