Hugging Face's logo

AbstractPhil
/
alephlm-0

Feature Extraction
TensorBoard
English
sentence-similarity
consensus-distillation
mixture-of-experts
sparse-routing
geometric-deep-learning
amoe
research-log
Model card Files
xet
 Metrics Community
AbstractPhil commited on
Commit
bc6c6c6
·
verified ·
1 Parent(s): 9ecc845

P6 mean-pooled variant renamed (launched without ALEPHLM_POOLING=cls; kept as the pooling-x-routing interaction row: mimicry .8620, capability .3801, toggle -.0092)

Browse files
Files changed (9) hide show
  1. runs/alephlm0-small-a3_random-mean-s0/checkpoints/best_model.pt +3 -0
  2. runs/alephlm0-small-a3_random-mean-s0/checkpoints/best_state.pt +3 -0
  3. runs/alephlm0-small-a3_random-mean-s0/checkpoints/final_model.pt +3 -0
  4. runs/alephlm0-small-a3_random-mean-s0/checkpoints/metrics.json +247 -0
  5. runs/alephlm0-small-a3_random-mean-s0/checkpoints/state.pt +3 -0
  6. runs/alephlm0-small-a3_random-mean-s0/checkpoints/tokenizer/tokenizer.json +0 -0
  7. runs/alephlm0-small-a3_random-mean-s0/checkpoints/tokenizer/tokenizer_config.json +15 -0
  8. runs/alephlm0-small-a3_random-mean-s0/config/config.json +94 -0
  9. runs/alephlm0-small-a3_random-mean-s0/tensorboard/alephlm0-small-a3_random-s0/events.out.tfevents.1786229128.Maximus-Omni.83288.0 +3 -0
runs/alephlm0-small-a3_random-mean-s0/checkpoints/best_model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2cb51fe7b4345fc4d8c980ba941f8cac6ecadca4feec40f70ba9c4805617f85e
3
+ size 233581023
runs/alephlm0-small-a3_random-mean-s0/checkpoints/best_state.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:39f9259d095a7a7fd31cb16079ee798fb64a57e0d98c1b91bfc2d7d8af1cd264
3
+ size 700479883
runs/alephlm0-small-a3_random-mean-s0/checkpoints/final_model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0d8fcb5673c3953fe85e7ed99cc3536a8a9456a20ceb3e9c9dcf45c043e18ba4
3
+ size 233581363
runs/alephlm0-small-a3_random-mean-s0/checkpoints/metrics.json ADDED
@@ -0,0 +1,247 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "profile": "_applied",
4
+ "run_name": "alephlm0-small-a3_random-s0",
5
+ "hf_repo": "AbstractPhil/alephlm-0",
6
+ "sources": [
7
+ {
8
+ "repo": "AbstractPhil/conceptual-captions-12m-webdataset-berts",
9
+ "n_chunks": 1,
10
+ "chunk_rows": 500000,
11
+ "missing": {}
12
+ }
13
+ ],
14
+ "experts": [
15
+ "bert",
16
+ "modern",
17
+ "roberta",
18
+ "albert",
19
+ "distil"
20
+ ],
21
+ "ref_expert": "bert",
22
+ "ref_hf_name": "google-bert/bert-base-uncased",
23
+ "require_all_experts": true,
24
+ "caption_field": "caption_llava",
25
+ "caption_field_candidates": [
26
+ "caption_llava",
27
+ "caption",
28
+ "caption_llava_short"
29
+ ],
30
+ "work_dir": "E:/mirel/data/alephlm0",
31
+ "keep_expert_shards": false,
32
+ "disk_floor_gb": 25.0,
33
+ "ram_resident": true,
34
+ "preflight": true,
35
+ "artifact_repo": "AbstractPhil/captionbert-8192-v2",
36
+ "targets_repo": "AbstractPhil/captionbert-8192-v2-consensus",
37
+ "push_targets": true,
38
+ "hf_push": true,
39
+ "push_every_min": 30.0,
40
+ "keep_local_ckpts": 3,
41
+ "parity_chunk": 0,
42
+ "parity_n": 64,
43
+ "parity_min_cos": 0.999,
44
+ "fit_chunks": [
45
+ 0,
46
+ 11,
47
+ 22,
48
+ 33,
49
+ 44,
50
+ 55
51
+ ],
52
+ "fit_rows_per_chunk": 4000,
53
+ "holdout_chunks": [
54
+ 60,
55
+ 61
56
+ ],
57
+ "fit_seed": 0,
58
+ "d_model": 512,
59
+ "n_heads": 8,
60
+ "n_layers": 12,
61
+ "d_ff": 2048,
62
+ "max_len": 8192,
63
+ "output_dim": 768,
64
+ "dropout": 0.1,
65
+ "pooling": "mean",
66
+ "max_tokens": 128,
67
+ "length_bucketing": true,
68
+ "bucket_window": 64,
69
+ "grad_checkpointing": true,
70
+ "vram_probe": true,
71
+ "epochs": 2,
72
+ "batch_size": 2048,
73
+ "lr": 0.0006,
74
+ "min_lr": 1e-06,
75
+ "warmup_steps": 100,
76
+ "grad_clip": 1.0,
77
+ "seed": 155979836,
78
+ "amp": true,
79
+ "num_workers": 0,
80
+ "log_every": 50,
81
+ "eval_every": 100,
82
+ "ckpt_every_min": 20.0,
83
+ "fresh_start": true,
84
+ "allow_mismatched_resume": false,
85
+ "nce_weight": 1.0,
86
+ "mse_weight": 1.0,
87
+ "nce_temperature": 0.07,
88
+ "cv_weight": 0.0,
89
+ "cv_target": 0.084,
90
+ "run_stage0": false,
91
+ "run_stage1": false,
92
+ "run_stage2": false,
93
+ "run_stage3": true,
94
+ "resume": true
95
+ },
96
+ "final": {
97
+ "mimicry_r1": 0.8619999885559082,
98
+ "cos_to_target": 0.6322637796401978,
99
+ "self_cos": 0.0028908364474773407,
100
+ "erank": 41.35349143289229,
101
+ "cv": 0.12663859327500038,
102
+ "n": 5120,
103
+ "frame_r1_after_rotation": 0.902999997138977,
104
+ "frame_cos_after_rotation": 0.650077760219574,
105
+ "frame_n_heldout": 2000,
106
+ "aleph_amp_by_block": [
107
+ 0.00628,
108
+ 0.00669,
109
+ 0.0069,
110
+ 0.00655,
111
+ 0.00631,
112
+ 0.00636,
113
+ 0.00649,
114
+ 0.00717,
115
+ 0.00744,
116
+ 0.00727,
117
+ 0.0076,
118
+ 0.00857
119
+ ],
120
+ "aleph_w_abs_by_block": [
121
+ 0.11016,
122
+ 0.1158,
123
+ 0.12111,
124
+ 0.11673,
125
+ 0.11789,
126
+ 0.11909,
127
+ 0.12402,
128
+ 0.13712,
129
+ 0.14084,
130
+ 0.13472,
131
+ 0.14334,
132
+ 0.15843
133
+ ],
134
+ "aleph_gate_sig": [
135
+ [
136
+ 0.0568,
137
+ 0.0591,
138
+ 0.0549
139
+ ],
140
+ [
141
+ 0.0566,
142
+ 0.058,
143
+ 0.0587
144
+ ],
145
+ [
146
+ 0.0561,
147
+ 0.0575,
148
+ 0.0573
149
+ ],
150
+ [
151
+ 0.0557,
152
+ 0.0556,
153
+ 0.0571
154
+ ],
155
+ [
156
+ 0.0533,
157
+ 0.0535,
158
+ 0.0536
159
+ ],
160
+ [
161
+ 0.0531,
162
+ 0.0537,
163
+ 0.0534
164
+ ],
165
+ [
166
+ 0.0525,
167
+ 0.0519,
168
+ 0.0525
169
+ ],
170
+ [
171
+ 0.0513,
172
+ 0.0525,
173
+ 0.0535
174
+ ],
175
+ [
176
+ 0.0535,
177
+ 0.0529,
178
+ 0.052
179
+ ],
180
+ [
181
+ 0.0536,
182
+ 0.0538,
183
+ 0.0545
184
+ ],
185
+ [
186
+ 0.0526,
187
+ 0.054,
188
+ 0.0522
189
+ ],
190
+ [
191
+ 0.0539,
192
+ 0.0547,
193
+ 0.0535
194
+ ]
195
+ ],
196
+ "aleph_drift_by_block": [
197
+ 0.0,
198
+ 0.00012,
199
+ 0.0,
200
+ 0.0,
201
+ 0.0,
202
+ 0.0,
203
+ 0.0,
204
+ 0.0,
205
+ 0.0,
206
+ 0.0,
207
+ 0.0,
208
+ 0.0
209
+ ],
210
+ "aleph_amp_b00": 0.00628,
211
+ "aleph_amp_b01": 0.00669,
212
+ "aleph_amp_b02": 0.0069,
213
+ "aleph_amp_b03": 0.00655,
214
+ "aleph_amp_b04": 0.00631,
215
+ "aleph_amp_b05": 0.00636,
216
+ "aleph_amp_b06": 0.00649,
217
+ "aleph_amp_b07": 0.00717,
218
+ "aleph_amp_b08": 0.00744,
219
+ "aleph_amp_b09": 0.00727,
220
+ "aleph_amp_b10": 0.0076,
221
+ "aleph_amp_b11": 0.00857,
222
+ "aleph_drift_b00": 0.0,
223
+ "aleph_drift_b01": 0.00012,
224
+ "aleph_drift_b02": 0.0,
225
+ "aleph_drift_b03": 0.0,
226
+ "aleph_drift_b04": 0.0,
227
+ "aleph_drift_b05": 0.0,
228
+ "aleph_drift_b06": 0.0,
229
+ "aleph_drift_b07": 0.0,
230
+ "aleph_drift_b08": 0.0,
231
+ "aleph_drift_b09": 0.0,
232
+ "aleph_drift_b10": 0.0,
233
+ "aleph_drift_b11": 0.0,
234
+ "aleph_gate_mean_b00": 0.056933333333333336,
235
+ "aleph_gate_mean_b01": 0.05776666666666667,
236
+ "aleph_gate_mean_b02": 0.056966666666666665,
237
+ "aleph_gate_mean_b03": 0.056133333333333334,
238
+ "aleph_gate_mean_b04": 0.05346666666666666,
239
+ "aleph_gate_mean_b05": 0.0534,
240
+ "aleph_gate_mean_b06": 0.05229999999999999,
241
+ "aleph_gate_mean_b07": 0.05243333333333333,
242
+ "aleph_gate_mean_b08": 0.05279999999999999,
243
+ "aleph_gate_mean_b09": 0.05396666666666666,
244
+ "aleph_gate_mean_b10": 0.05293333333333333,
245
+ "aleph_gate_mean_b11": 0.05403333333333333
246
+ }
247
+ }
runs/alephlm0-small-a3_random-mean-s0/checkpoints/state.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0b94d7b48c0da3884798527a36e40488593d72e60eaf44610beae3cd4af06d93
3
+ size 700425656
runs/alephlm0-small-a3_random-mean-s0/checkpoints/tokenizer/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
runs/alephlm0-small-a3_random-mean-s0/checkpoints/tokenizer/tokenizer_config.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "cls_token": "[CLS]",
4
+ "do_lower_case": true,
5
+ "is_local": false,
6
+ "local_files_only": false,
7
+ "mask_token": "[MASK]",
8
+ "model_max_length": 512,
9
+ "pad_token": "[PAD]",
10
+ "sep_token": "[SEP]",
11
+ "strip_accents": null,
12
+ "tokenize_chinese_chars": true,
13
+ "tokenizer_class": "BertTokenizer",
14
+ "unk_token": "[UNK]"
15
+ }
runs/alephlm0-small-a3_random-mean-s0/config/config.json ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "profile": "_applied",
3
+ "run_name": "alephlm0-small-a3_random-s0",
4
+ "hf_repo": "AbstractPhil/alephlm-0",
5
+ "sources": [
6
+ {
7
+ "repo": "AbstractPhil/conceptual-captions-12m-webdataset-berts",
8
+ "n_chunks": 1,
9
+ "chunk_rows": 500000,
10
+ "missing": {}
11
+ }
12
+ ],
13
+ "experts": [
14
+ "bert",
15
+ "modern",
16
+ "roberta",
17
+ "albert",
18
+ "distil"
19
+ ],
20
+ "ref_expert": "bert",
21
+ "ref_hf_name": "google-bert/bert-base-uncased",
22
+ "require_all_experts": true,
23
+ "caption_field": "caption_llava",
24
+ "caption_field_candidates": [
25
+ "caption_llava",
26
+ "caption",
27
+ "caption_llava_short"
28
+ ],
29
+ "work_dir": "E:/mirel/data/alephlm0",
30
+ "keep_expert_shards": false,
31
+ "disk_floor_gb": 25.0,
32
+ "ram_resident": true,
33
+ "preflight": true,
34
+ "artifact_repo": "AbstractPhil/captionbert-8192-v2",
35
+ "targets_repo": "AbstractPhil/captionbert-8192-v2-consensus",
36
+ "push_targets": true,
37
+ "hf_push": true,
38
+ "push_every_min": 30.0,
39
+ "keep_local_ckpts": 3,
40
+ "parity_chunk": 0,
41
+ "parity_n": 64,
42
+ "parity_min_cos": 0.999,
43
+ "fit_chunks": [
44
+ 0,
45
+ 11,
46
+ 22,
47
+ 33,
48
+ 44,
49
+ 55
50
+ ],
51
+ "fit_rows_per_chunk": 4000,
52
+ "holdout_chunks": [
53
+ 60,
54
+ 61
55
+ ],
56
+ "fit_seed": 0,
57
+ "d_model": 512,
58
+ "n_heads": 8,
59
+ "n_layers": 12,
60
+ "d_ff": 2048,
61
+ "max_len": 8192,
62
+ "output_dim": 768,
63
+ "dropout": 0.1,
64
+ "pooling": "mean",
65
+ "max_tokens": 128,
66
+ "length_bucketing": true,
67
+ "bucket_window": 64,
68
+ "grad_checkpointing": true,
69
+ "vram_probe": true,
70
+ "epochs": 2,
71
+ "batch_size": 2048,
72
+ "lr": 0.0006,
73
+ "min_lr": 1e-06,
74
+ "warmup_steps": 100,
75
+ "grad_clip": 1.0,
76
+ "seed": 155979836,
77
+ "amp": true,
78
+ "num_workers": 0,
79
+ "log_every": 50,
80
+ "eval_every": 100,
81
+ "ckpt_every_min": 20.0,
82
+ "fresh_start": true,
83
+ "allow_mismatched_resume": false,
84
+ "nce_weight": 1.0,
85
+ "mse_weight": 1.0,
86
+ "nce_temperature": 0.07,
87
+ "cv_weight": 0.0,
88
+ "cv_target": 0.084,
89
+ "run_stage0": false,
90
+ "run_stage1": false,
91
+ "run_stage2": false,
92
+ "run_stage3": true,
93
+ "resume": true
94
+ }
runs/alephlm0-small-a3_random-mean-s0/tensorboard/alephlm0-small-a3_random-s0/events.out.tfevents.1786229128.Maximus-Omni.83288.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d777be3cdede59ba86c50d60880d731fca8449ca240d45585c501cdc88389daf
3
+ size 17645