Hugging Face's logo

AbstractPhil
/
alephlm-0

Feature Extraction
TensorBoard
English
sentence-similarity
consensus-distillation
mixture-of-experts
sparse-routing
geometric-deep-learning
amoe
research-log
Model card Files
xet
 Metrics Community
AbstractPhil commited on
Commit
a911967
·
verified ·
1 Parent(s): b6d022d

final (runs/alephlm0-small-a4_splat-cbk-gf-wa-s1-e6-gov25m/checkpoints)

Browse files
Files changed (5) hide show
  1. runs/alephlm0-small-a4_splat-cbk-gf-wa-s1-e6-gov25m/checkpoints/final_model.pt +3 -0
  2. runs/alephlm0-small-a4_splat-cbk-gf-wa-s1-e6-gov25m/checkpoints/metrics.json +295 -0
  3. runs/alephlm0-small-a4_splat-cbk-gf-wa-s1-e6-gov25m/checkpoints/state.pt +1 -1
  4. runs/alephlm0-small-a4_splat-cbk-gf-wa-s1-e6-gov25m/checkpoints/tokenizer/tokenizer.json +0 -0
  5. runs/alephlm0-small-a4_splat-cbk-gf-wa-s1-e6-gov25m/checkpoints/tokenizer/tokenizer_config.json +15 -0
runs/alephlm0-small-a4_splat-cbk-gf-wa-s1-e6-gov25m/checkpoints/final_model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:56b01f29a5f2106c31123d9e9ba57920a49d377b81ed5ca92012cb69c355011c
3
+ size 297572065
runs/alephlm0-small-a4_splat-cbk-gf-wa-s1-e6-gov25m/checkpoints/metrics.json ADDED
@@ -0,0 +1,295 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "profile": "_applied",
4
+ "run_name": "alephlm0-small-a4_splat-cbk-gf-wa-s1-e6-gov25m",
5
+ "hf_repo": "AbstractPhil/alephlm-0",
6
+ "sources": [
7
+ {
8
+ "repo": "AbstractPhil/conceptual-captions-12m-webdataset-berts",
9
+ "n_chunks": 1,
10
+ "chunk_rows": 500000,
11
+ "missing": {}
12
+ }
13
+ ],
14
+ "experts": [
15
+ "bert",
16
+ "modern",
17
+ "roberta",
18
+ "albert",
19
+ "distil"
20
+ ],
21
+ "ref_expert": "bert",
22
+ "ref_hf_name": "google-bert/bert-base-uncased",
23
+ "require_all_experts": true,
24
+ "caption_field": "caption_llava",
25
+ "caption_field_candidates": [
26
+ "caption_llava",
27
+ "caption",
28
+ "caption_llava_short"
29
+ ],
30
+ "work_dir": "E:/mirel/data/alephlm0",
31
+ "keep_expert_shards": false,
32
+ "disk_floor_gb": 25.0,
33
+ "ram_resident": true,
34
+ "preflight": true,
35
+ "artifact_repo": "AbstractPhil/captionbert-8192-v2",
36
+ "targets_repo": "AbstractPhil/captionbert-8192-v2-consensus",
37
+ "push_targets": true,
38
+ "hf_push": true,
39
+ "push_every_min": 30.0,
40
+ "keep_local_ckpts": 3,
41
+ "parity_chunk": 0,
42
+ "parity_n": 64,
43
+ "parity_min_cos": 0.999,
44
+ "fit_chunks": [
45
+ 0,
46
+ 11,
47
+ 22,
48
+ 33,
49
+ 44,
50
+ 55
51
+ ],
52
+ "fit_rows_per_chunk": 4000,
53
+ "holdout_chunks": [
54
+ 60,
55
+ 61
56
+ ],
57
+ "fit_seed": 0,
58
+ "d_model": 512,
59
+ "n_heads": 8,
60
+ "n_layers": 12,
61
+ "d_ff": 2048,
62
+ "max_len": 8192,
63
+ "output_dim": 768,
64
+ "dropout": 0.1,
65
+ "pooling": "cls",
66
+ "max_tokens": 128,
67
+ "length_bucketing": true,
68
+ "bucket_window": 64,
69
+ "grad_checkpointing": true,
70
+ "vram_probe": true,
71
+ "epochs": 6,
72
+ "batch_size": 2048,
73
+ "lr": 0.0006,
74
+ "min_lr": 1e-06,
75
+ "warmup_steps": 100,
76
+ "grad_clip": 1.0,
77
+ "seed": 46212409,
78
+ "amp": true,
79
+ "num_workers": 0,
80
+ "log_every": 50,
81
+ "eval_every": 100,
82
+ "ckpt_every_min": 20.0,
83
+ "fresh_start": true,
84
+ "allow_mismatched_resume": false,
85
+ "nce_weight": 1.0,
86
+ "mse_weight": 1.0,
87
+ "nce_temperature": 0.07,
88
+ "cv_weight": 0.0,
89
+ "cv_target": 0.084,
90
+ "run_stage0": false,
91
+ "run_stage1": false,
92
+ "run_stage2": false,
93
+ "run_stage3": true,
94
+ "resume": true
95
+ },
96
+ "final": {
97
+ "mimicry_r1": 0.847000002861023,
98
+ "cos_to_target": 0.6297791004180908,
99
+ "self_cos": 0.010518290102481842,
100
+ "erank": 40.513665627948605,
101
+ "cv": 0.14081885812084063,
102
+ "n": 5120,
103
+ "frame_r1_after_rotation": 0.8824999928474426,
104
+ "frame_cos_after_rotation": 0.6482672691345215,
105
+ "frame_n_heldout": 2000,
106
+ "aleph_amp_by_block": [
107
+ 0.00547,
108
+ 0.00837,
109
+ 0.00992,
110
+ 0.0092,
111
+ 0.00964,
112
+ 0.00927,
113
+ 0.00881,
114
+ 0.01033,
115
+ 0.0096,
116
+ 0.01025,
117
+ 0.0101,
118
+ 0.0093
119
+ ],
120
+ "aleph_w_abs_by_block": [
121
+ 0.09963,
122
+ 0.16337,
123
+ 0.19705,
124
+ 0.18516,
125
+ 0.19205,
126
+ 0.18839,
127
+ 0.18019,
128
+ 0.20795,
129
+ 0.18426,
130
+ 0.19487,
131
+ 0.19575,
132
+ 0.18957
133
+ ],
134
+ "aleph_gate_sig": [
135
+ [
136
+ 0.0528,
137
+ 0.0555,
138
+ 0.0567
139
+ ],
140
+ [
141
+ 0.0552,
142
+ 0.0493,
143
+ 0.0488
144
+ ],
145
+ [
146
+ 0.0499,
147
+ 0.0504,
148
+ 0.0507
149
+ ],
150
+ [
151
+ 0.0485,
152
+ 0.0497,
153
+ 0.0505
154
+ ],
155
+ [
156
+ 0.0499,
157
+ 0.0515,
158
+ 0.0489
159
+ ],
160
+ [
161
+ 0.0487,
162
+ 0.0501,
163
+ 0.0486
164
+ ],
165
+ [
166
+ 0.0479,
167
+ 0.0499,
168
+ 0.0491
169
+ ],
170
+ [
171
+ 0.0513,
172
+ 0.0488,
173
+ 0.0495
174
+ ],
175
+ [
176
+ 0.051,
177
+ 0.0534,
178
+ 0.0514
179
+ ],
180
+ [
181
+ 0.0531,
182
+ 0.0527,
183
+ 0.0522
184
+ ],
185
+ [
186
+ 0.0538,
187
+ 0.0504,
188
+ 0.0507
189
+ ],
190
+ [
191
+ 0.0505,
192
+ 0.0476,
193
+ 0.049
194
+ ]
195
+ ],
196
+ "aleph_drift_by_block": [
197
+ 0.59788,
198
+ 0.45638,
199
+ 0.50847,
200
+ 0.41118,
201
+ 0.44945,
202
+ 0.44151,
203
+ 0.40213,
204
+ 0.50442,
205
+ 0.50454,
206
+ 0.63954,
207
+ 0.56487,
208
+ 0.53142
209
+ ],
210
+ "splat_mass_abs_mean_b00": 0.14917,
211
+ "splat_mass_abs_max_b00": 0.60816,
212
+ "splat_mass_net_mean_b00": 0.005,
213
+ "splat_frac_neg_b00": 0.4609,
214
+ "splat_mass_abs_mean_b01": 0.15217,
215
+ "splat_mass_abs_max_b01": 0.60157,
216
+ "splat_mass_net_mean_b01": 0.00425,
217
+ "splat_frac_neg_b01": 0.4609,
218
+ "splat_mass_abs_mean_b02": 0.15399,
219
+ "splat_mass_abs_max_b02": 0.64881,
220
+ "splat_mass_net_mean_b02": -0.00364,
221
+ "splat_frac_neg_b02": 0.5625,
222
+ "splat_mass_abs_mean_b03": 0.15082,
223
+ "splat_mass_abs_max_b03": 0.68647,
224
+ "splat_mass_net_mean_b03": -0.00163,
225
+ "splat_frac_neg_b03": 0.5703,
226
+ "splat_mass_abs_mean_b04": 0.14917,
227
+ "splat_mass_abs_max_b04": 0.67446,
228
+ "splat_mass_net_mean_b04": 0.00756,
229
+ "splat_frac_neg_b04": 0.4688,
230
+ "splat_mass_abs_mean_b05": 0.14635,
231
+ "splat_mass_abs_max_b05": 0.73375,
232
+ "splat_mass_net_mean_b05": 0.00557,
233
+ "splat_frac_neg_b05": 0.4844,
234
+ "splat_mass_abs_mean_b06": 0.14607,
235
+ "splat_mass_abs_max_b06": 0.64825,
236
+ "splat_mass_net_mean_b06": -0.00072,
237
+ "splat_frac_neg_b06": 0.5078,
238
+ "splat_mass_abs_mean_b07": 0.15437,
239
+ "splat_mass_abs_max_b07": 0.70107,
240
+ "splat_mass_net_mean_b07": 0.01056,
241
+ "splat_frac_neg_b07": 0.4609,
242
+ "splat_mass_abs_mean_b08": 0.15405,
243
+ "splat_mass_abs_max_b08": 0.73924,
244
+ "splat_mass_net_mean_b08": 0.01015,
245
+ "splat_frac_neg_b08": 0.5156,
246
+ "splat_mass_abs_mean_b09": 0.15008,
247
+ "splat_mass_abs_max_b09": 0.68234,
248
+ "splat_mass_net_mean_b09": 0.00014,
249
+ "splat_frac_neg_b09": 0.5703,
250
+ "splat_mass_abs_mean_b10": 0.15173,
251
+ "splat_mass_abs_max_b10": 0.78181,
252
+ "splat_mass_net_mean_b10": -0.01042,
253
+ "splat_frac_neg_b10": 0.4844,
254
+ "splat_mass_abs_mean_b11": 0.15493,
255
+ "splat_mass_abs_max_b11": 0.9432,
256
+ "splat_mass_net_mean_b11": 0.0015,
257
+ "splat_frac_neg_b11": 0.5,
258
+ "aleph_amp_b00": 0.00547,
259
+ "aleph_amp_b01": 0.00837,
260
+ "aleph_amp_b02": 0.00992,
261
+ "aleph_amp_b03": 0.0092,
262
+ "aleph_amp_b04": 0.00964,
263
+ "aleph_amp_b05": 0.00927,
264
+ "aleph_amp_b06": 0.00881,
265
+ "aleph_amp_b07": 0.01033,
266
+ "aleph_amp_b08": 0.0096,
267
+ "aleph_amp_b09": 0.01025,
268
+ "aleph_amp_b10": 0.0101,
269
+ "aleph_amp_b11": 0.0093,
270
+ "aleph_drift_b00": 0.59788,
271
+ "aleph_drift_b01": 0.45638,
272
+ "aleph_drift_b02": 0.50847,
273
+ "aleph_drift_b03": 0.41118,
274
+ "aleph_drift_b04": 0.44945,
275
+ "aleph_drift_b05": 0.44151,
276
+ "aleph_drift_b06": 0.40213,
277
+ "aleph_drift_b07": 0.50442,
278
+ "aleph_drift_b08": 0.50454,
279
+ "aleph_drift_b09": 0.63954,
280
+ "aleph_drift_b10": 0.56487,
281
+ "aleph_drift_b11": 0.53142,
282
+ "aleph_gate_mean_b00": 0.055,
283
+ "aleph_gate_mean_b01": 0.0511,
284
+ "aleph_gate_mean_b02": 0.050333333333333334,
285
+ "aleph_gate_mean_b03": 0.04956666666666667,
286
+ "aleph_gate_mean_b04": 0.0501,
287
+ "aleph_gate_mean_b05": 0.049133333333333334,
288
+ "aleph_gate_mean_b06": 0.048966666666666665,
289
+ "aleph_gate_mean_b07": 0.04986666666666667,
290
+ "aleph_gate_mean_b08": 0.05193333333333333,
291
+ "aleph_gate_mean_b09": 0.05266666666666667,
292
+ "aleph_gate_mean_b10": 0.05163333333333334,
293
+ "aleph_gate_mean_b11": 0.04903333333333334
294
+ }
295
+ }
runs/alephlm0-small-a4_splat-cbk-gf-wa-s1-e6-gov25m/checkpoints/state.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4dc8377287f00cc08be9c48c873283384077fe69bb4cd148b71a706ee736576d
3
  size 790917976
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:03b9ac0186fea99702597ec0df393e2bd8621f86e21eba81ee7552e6cda80f3a
3
  size 790917976
runs/alephlm0-small-a4_splat-cbk-gf-wa-s1-e6-gov25m/checkpoints/tokenizer/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
runs/alephlm0-small-a4_splat-cbk-gf-wa-s1-e6-gov25m/checkpoints/tokenizer/tokenizer_config.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "cls_token": "[CLS]",
4
+ "do_lower_case": true,
5
+ "is_local": false,
6
+ "local_files_only": false,
7
+ "mask_token": "[MASK]",
8
+ "model_max_length": 512,
9
+ "pad_token": "[PAD]",
10
+ "sep_token": "[SEP]",
11
+ "strip_accents": null,
12
+ "tokenize_chinese_chars": true,
13
+ "tokenizer_class": "BertTokenizer",
14
+ "unk_token": "[UNK]"
15
+ }