suchirsalhan commited on
Commit
50df588
·
verified ·
1 Parent(s): 37807f1

run records: final.json, log, probes

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
config.yaml ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model: {architecture: hnet, d_enc: 256, d_main: 512, n_enc: 2, n_dec: 2, n_main: 6,
2
+ n_heads_enc: 4, n_heads_main: 8, seq_len: 1024, dropout: 0.0, target_ratio: 6.0,
3
+ max_chunk_mult: 3.0, dechunk: ste, router_noise: 0.0, rope_base: 10000.0}
4
+ memory:
5
+ ali:
6
+ budget_B: 65536
7
+ staging_capacity: 262144
8
+ policy: utility
9
+ candidate_types: [surface, split, merge, skel]
10
+ write_rule: mean
11
+ eta_min: 0.0
12
+ memory_scope: persistent
13
+ consolidate_every: 50
14
+ max_span: 64
15
+ gate: vector
16
+ retrieval: true
17
+ read_dropout: 0.1
18
+ gate_hidden: 0
19
+ type_dim: 16
20
+ gate_bias_init: -2.0
21
+ probe_seqs: 2
22
+ probe_reps: 8
23
+ horizon: 32
24
+ predictor: mlp
25
+ lambda_util: 0.1
26
+ enabled: true
27
+ inventory_kwargs: {}
28
+ probe_mix: null
29
+ util_batch_slots: 1024
30
+ pkm: {n_keys: 256, topk: 8, heads: 1, d_key: null, value_lr_mult: 1.0}
31
+ objective: {name: parity, schedule: B, alpha: 0.03, alpha_parity_cap: 0.06, global_anchor: 0.0,
32
+ premium_clip: 4.0, premium_kind: flores_per_sent, tau1: 0.4, tau2: 0.7, freeze_scope: router,
33
+ cpc_decay: 0.9, genreg_target_entropy: null, genreg_eta: 0.05}
34
+ data: {corpus: corpus, langs: null, mixture: null, doc_filters: null, byte_classes: scratch/data/byte_classes.json}
35
+ training:
36
+ seed: 0
37
+ max_steps: 6000
38
+ batch_size: 32
39
+ grad_accum: 1
40
+ lr: 0.0003
41
+ min_lr_frac: 0.1
42
+ warmup: 300
43
+ weight_decay: 0.1
44
+ grad_clip: 1.0
45
+ betas: [0.9, 0.95]
46
+ router_lr_mult: 1.0
47
+ log_every: 50
48
+ amp: bf16
49
+ compile: false
50
+ device: cuda
51
+ checkpointing:
52
+ run_name: parity_B_ste_s0
53
+ runs_dir: runs
54
+ ckpt_every: 200
55
+ keep: false
56
+ milestones: [1000, 3000]
57
+ resume: auto
58
+ evaluation:
59
+ eval_every: 200
60
+ eval_batches: 8
61
+ probe_sents: 128
62
+ probe_bpb_batches: 4
63
+ log_spaced_probe: true
64
+ dense_early_every: 0
65
+ dense_early_until: 0
66
+ dump_reps: true
67
+ dump_reps_thresholds: [0.3, 0.4, 0.5, 0.6, 0.7]
final.json ADDED
@@ -0,0 +1,120 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "event": "final",
3
+ "run": "parity_B_ste_s0",
4
+ "arch": "hnet",
5
+ "objective": "parity",
6
+ "condition": "B",
7
+ "step": 6000,
8
+ "params": 22516992,
9
+ "seed": 0,
10
+ "per_lang": {
11
+ "en": {
12
+ "cpc": 0.26110440728610007,
13
+ "cps": 35.5,
14
+ "bpb": 2.0320761994873697,
15
+ "bytes_per_chunk": 3.8016953208275024
16
+ },
17
+ "de": {
18
+ "cpc": 0.29401451322261724,
19
+ "cps": 47.1640625,
20
+ "bpb": 2.206460846043454,
21
+ "bytes_per_chunk": 3.425196644628531
22
+ },
23
+ "vi": {
24
+ "cpc": 0.2769189042582045,
25
+ "cps": 39.8828125,
26
+ "bpb": 1.9191335373327487,
27
+ "bytes_per_chunk": 4.586144156752974
28
+ },
29
+ "tr": {
30
+ "cpc": 0.33082914434071703,
31
+ "cps": 46.7890625,
32
+ "bpb": 2.1688446609108367,
33
+ "bytes_per_chunk": 3.227639023128967
34
+ },
35
+ "fi": {
36
+ "cpc": 0.2988676838020307,
37
+ "cps": 43.921875,
38
+ "bpb": 2.378471777303414,
39
+ "bytes_per_chunk": 3.554856426077228
40
+ },
41
+ "ru": {
42
+ "cpc": 0.3059630606860158,
43
+ "cps": 45.296875,
44
+ "bpb": 1.3591301434361727,
45
+ "bytes_per_chunk": 5.815406799401032
46
+ },
47
+ "el": {
48
+ "cpc": 0.3075086488330166,
49
+ "cps": 49.3046875,
50
+ "bpb": 1.4236893295078978,
51
+ "bytes_per_chunk": 6.0818746012412275
52
+ },
53
+ "hi": {
54
+ "cpc": 0.20855869242199107,
55
+ "cps": 27.4140625,
56
+ "bpb": 1.1790276918015519,
57
+ "bytes_per_chunk": 11.712530438085025
58
+ },
59
+ "ta": {
60
+ "cpc": 0.2545600427921904,
61
+ "cps": 37.1796875,
62
+ "bpb": 0.9676912290931334,
63
+ "bytes_per_chunk": 10.479262857028642
64
+ },
65
+ "th": {
66
+ "cpc": 0.30556419023935344,
67
+ "cps": 38.3984375,
68
+ "bpb": 1.0954840449550585,
69
+ "bytes_per_chunk": 9.267951210889164
70
+ },
71
+ "zh": {
72
+ "cpc": 0.6285133762275652,
73
+ "cps": 29.0,
74
+ "bpb": 3.125566476443265,
75
+ "bytes_per_chunk": 4.495888179050723
76
+ },
77
+ "my": {
78
+ "cpc": 0.251002744352966,
79
+ "cps": 37.15625,
80
+ "bpb": 1.1037527612046865,
81
+ "bytes_per_chunk": 10.623655042451015
82
+ }
83
+ },
84
+ "bpb_mean": 1.746610724793299,
85
+ "gini_cpc": 0.13481524300909165,
86
+ "gini_cps": 0.09533577913534352,
87
+ "bpic": 6.2856183566175865,
88
+ "mean_boundary_entropy": 0.6647466421127319,
89
+ "dechunk": "ste",
90
+ "router_noise": 0.0,
91
+ "transitions": [
92
+ {
93
+ "step": 0,
94
+ "phase": 1,
95
+ "objective": "global",
96
+ "router_trainable": true,
97
+ "encoder_trainable": true,
98
+ "frozen_router_params": 0
99
+ },
100
+ {
101
+ "step": 2400,
102
+ "phase": 2,
103
+ "objective": "parity_a",
104
+ "router_trainable": true,
105
+ "encoder_trainable": true,
106
+ "frozen_router_params": 0
107
+ },
108
+ {
109
+ "step": 4200,
110
+ "phase": 3,
111
+ "objective": "none",
112
+ "router_trainable": false,
113
+ "encoder_trainable": true,
114
+ "frozen_router_params": 131584
115
+ }
116
+ ],
117
+ "probe_overhead_sec_total": 0.0,
118
+ "probe_overhead_seq_frac": 0.0,
119
+ "wall_sec": 445.2848217487335
120
+ }
log.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
probe/000000__de.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d12734bfb73323c61d25be0fadd7b6c98aab0383557aeb4e17119804914f039a
3
+ size 306038
probe/000000__el.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bc88d34544806f8d8d4f17f1d9d001518b381aa35d403845bf543083b6661f59
3
+ size 483837
probe/000000__en.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e10a1bf7b1e92b09ae71d52c484b6a0dffee17a335498039da7bebafdae9c043
3
+ size 258090
probe/000000__fi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3fc3a9ea7870a853308c1c2fd3d867046ba99740ae29c63bf0d68003b5ee9622
3
+ size 295839
probe/000000__hi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e9e063ef44816e7f4831461764732fda11546717fd4e79ac1fc05f23aed24aec
3
+ size 494957
probe/000000__my.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1659586644f31f62b916ab708eba8904aec8b8d95463b69a629e35a463094f48
3
+ size 572569
probe/000000__ru.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e1c31bd2b7cdc50bf129e378f7888da5535ad653d6233012a92ba779185a3ef4
3
+ size 459298
probe/000000__ta.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b374bd18e161ccc252a8ee2f2b603c747d0c178a219fbc03a33963e3f06ef1fe
3
+ size 569893
probe/000000__th.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bd10944073129d2bcd422e063be6be8bf9c2cad7833473e2445e48c4703f6d43
3
+ size 528670
probe/000000__tr.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b17913ef0aaa7fa9af59182112455148fd422c69e9207c26d4f8a1eceb7ca7a2
3
+ size 279476
probe/000000__vi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a17800080c7d75a989f1124eacd6272709a5c26a4ad3e63a6bb681e02e0e7d8d
3
+ size 350139
probe/000000__zh.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8d5b028fff9162d9460a55dd1fbb9dc90c5f7b1826b2e911bce6c8647d06c0d9
3
+ size 232843
probe/000001__de.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ac10ff216ab7052ed7ac42042723e3c9573ff2b6323966b75c91d135d5f4b617
3
+ size 306010
probe/000001__el.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:425087483019608976026383bcf24c9ed904aadb9a8110725da72f42d027f097
3
+ size 483865
probe/000001__en.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7625d2a8b62f7c1d71b4d1a2bd0347d3ba9f4541b99fc03f9213ab454be278aa
3
+ size 258174
probe/000001__fi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:098419180c34b000a3efb1e73cd9d9803c85a1fbbd1b777cd9d8c3cb709ed576
3
+ size 295808
probe/000001__hi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b351488b46c66a5bb198d6a2bb8aeca3cf337482941206dab8ac71dbd008303e
3
+ size 494620
probe/000001__my.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d2c30055c40cb7b208c6ee08e49b3d7e9566cadd1afefc1d50aae10c26a5fa00
3
+ size 572558
probe/000001__ru.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4967ee3b56b5bda3e0a7f51003dd12576bebded662b3e668e0c73aff3357e76c
3
+ size 459364
probe/000001__ta.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9adc9256ca01fcec5d0cbd94da9009c6a1cb04d4fa17699b37cfb6681ede1f6
3
+ size 570093
probe/000001__th.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bfd2b791e2ee9ce2d98d5c08c988e3dbe4f288a744b101ac1589b9a64f4e180e
3
+ size 528404
probe/000001__tr.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a13bc035dc7e18cfe79a04eba94dc1fed6f66ff15274ead917ef1666b6e1034b
3
+ size 279583
probe/000001__vi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4fca82d377e81a91ce073cd25ac4fd77a1066883d1413e2ff32a0553ce14eeda
3
+ size 350139
probe/000001__zh.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e6b4e271b795ecf56e001528bee7659823f62905407d48632bfb23fddbe042ec
3
+ size 232940
probe/000002__de.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1633e868c6a614c36339ccae74093c963cfa305123bbe593520504e18b5d613b
3
+ size 305955
probe/000002__el.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4aa9179bd5fc6a4c3852e76fd14f344c2cb79d6ba56b2251a89f12b8abdd325d
3
+ size 483836
probe/000002__en.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:12fc3fe341e06699c679e541c5b90454d8290fadf458529c9e09a35677b9b6e9
3
+ size 258059
probe/000002__fi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5bbb84a058e00fb19809d0405e2e82fbcb368eb7d71b4fb536a88a689fcbfae5
3
+ size 295754
probe/000002__hi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:672660ae50f44bc4421eda6691560174882cd3c0c461ea3feaabbd375b3e0ad0
3
+ size 494335
probe/000002__my.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0e9cd1fc73006c0c2a90a43f99c1166b539479997dea6918d0075e3ed8d5b21
3
+ size 572104
probe/000002__ru.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7d1ad08ba26d8463d321cbe2340ac44c4f3512633bdebabf591539a1909eb361
3
+ size 459327
probe/000002__ta.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cb5d0fe6086cc3114c692f4f8ce0506db2938f54884307283cf832fc48fa2c6c
3
+ size 570597
probe/000002__th.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:55c1305df31c5e6e7f1b19bd34806b9a3944b9eb60a68ff61d388af20510bb2b
3
+ size 528990
probe/000002__tr.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:953b99856eb717636897dbfe2afda81413f7aada74494ed05e39142ebc01ed24
3
+ size 279383
probe/000002__vi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:72ac957c8ff865fb13448cc71bb806bdbf2ed6915c86f43e556c1006b3db71be
3
+ size 350381
probe/000002__zh.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b64aa0f6b4e0c677935adb022f449562ddefceddb3f669882e96d06e4ff36118
3
+ size 232948
probe/000004__de.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5b84d136e098f8b9ac08a7b32be0f9531aa23c302e27978c406d1025e0a6b902
3
+ size 305933
probe/000004__el.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:626b1afc23ff041a6ff14732ec28ba9d74fea256ced31c284b532d0707af9fb1
3
+ size 483599
probe/000004__en.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:943c3d7dae29b101e057a1c3c4665edff5569a960a3ab1ca717379bc642dbba3
3
+ size 258043
probe/000004__fi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4b46aae3b7a355ede16ac02465c34e443ed4df1ba9a30c5af8780e6f071b425b
3
+ size 295685
probe/000004__hi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0bc2c5c2e9abce33c92003a096995d23ccd08998d71fd307310c86c2823246f
3
+ size 493527
probe/000004__my.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:396706d6b39e99a27e618209060270af9a6d37f5b118f5b6aee3f86ff5d18d08
3
+ size 572042
probe/000004__ru.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d5e67fab6e209755c565b11c7fa226e109d51dffd49b142a61ff55114211c6cf
3
+ size 459151
probe/000004__ta.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aad3675181c00e796e1267472f8f15b80891005eb7405985847010b320798104
3
+ size 571813
probe/000004__th.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d255e2598c0f3b784e98bcd8dd51f43eda99705867697b83b30a8a2d72056772
3
+ size 529532
probe/000004__tr.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:69da581f46060afcc88efb324d44731201e332e3854ee10b7d2083e08a723921
3
+ size 279201
probe/000004__vi.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d3540d370d9b0279220a047c251fd06645c437984023c8ba576ec8b373813983
3
+ size 350388