Hugging Face's logo

AbstractPhil
/
alephlm-0

Feature Extraction
TensorBoard
English
sentence-similarity
consensus-distillation
mixture-of-experts
sparse-routing
geometric-deep-learning
amoe
research-log
Model card Files
xet
 Metrics Community
AbstractPhil commited on
Commit
55a8f32
·
verified ·
1 Parent(s): de7418c

final (runs/alephlm0-small-a4_splat-cbk-s0/checkpoints)

Browse files
Files changed (5) hide show
  1. runs/alephlm0-small-a4_splat-cbk-s0/checkpoints/final_model.pt +3 -0
  2. runs/alephlm0-small-a4_splat-cbk-s0/checkpoints/metrics.json +295 -0
  3. runs/alephlm0-small-a4_splat-cbk-s0/checkpoints/state.pt +1 -1
  4. runs/alephlm0-small-a4_splat-cbk-s0/checkpoints/tokenizer/tokenizer.json +0 -0
  5. runs/alephlm0-small-a4_splat-cbk-s0/checkpoints/tokenizer/tokenizer_config.json +15 -0
runs/alephlm0-small-a4_splat-cbk-s0/checkpoints/final_model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aa86cbf18836f1f9e51020e1fd709ef3208b95098846e0f2e3f35da413457092
3
+ size 284984573
runs/alephlm0-small-a4_splat-cbk-s0/checkpoints/metrics.json ADDED
@@ -0,0 +1,295 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "profile": "_applied",
4
+ "run_name": "alephlm0-small-a4_splat-cbk-s0",
5
+ "hf_repo": "AbstractPhil/alephlm-0",
6
+ "sources": [
7
+ {
8
+ "repo": "AbstractPhil/conceptual-captions-12m-webdataset-berts",
9
+ "n_chunks": 1,
10
+ "chunk_rows": 500000,
11
+ "missing": {}
12
+ }
13
+ ],
14
+ "experts": [
15
+ "bert",
16
+ "modern",
17
+ "roberta",
18
+ "albert",
19
+ "distil"
20
+ ],
21
+ "ref_expert": "bert",
22
+ "ref_hf_name": "google-bert/bert-base-uncased",
23
+ "require_all_experts": true,
24
+ "caption_field": "caption_llava",
25
+ "caption_field_candidates": [
26
+ "caption_llava",
27
+ "caption",
28
+ "caption_llava_short"
29
+ ],
30
+ "work_dir": "E:/mirel/data/alephlm0",
31
+ "keep_expert_shards": false,
32
+ "disk_floor_gb": 25.0,
33
+ "ram_resident": true,
34
+ "preflight": true,
35
+ "artifact_repo": "AbstractPhil/captionbert-8192-v2",
36
+ "targets_repo": "AbstractPhil/captionbert-8192-v2-consensus",
37
+ "push_targets": true,
38
+ "hf_push": true,
39
+ "push_every_min": 30.0,
40
+ "keep_local_ckpts": 3,
41
+ "parity_chunk": 0,
42
+ "parity_n": 64,
43
+ "parity_min_cos": 0.999,
44
+ "fit_chunks": [
45
+ 0,
46
+ 11,
47
+ 22,
48
+ 33,
49
+ 44,
50
+ 55
51
+ ],
52
+ "fit_rows_per_chunk": 4000,
53
+ "holdout_chunks": [
54
+ 60,
55
+ 61
56
+ ],
57
+ "fit_seed": 0,
58
+ "d_model": 512,
59
+ "n_heads": 8,
60
+ "n_layers": 12,
61
+ "d_ff": 2048,
62
+ "max_len": 8192,
63
+ "output_dim": 768,
64
+ "dropout": 0.1,
65
+ "pooling": "cls",
66
+ "max_tokens": 128,
67
+ "length_bucketing": true,
68
+ "bucket_window": 64,
69
+ "grad_checkpointing": true,
70
+ "vram_probe": true,
71
+ "epochs": 2,
72
+ "batch_size": 2048,
73
+ "lr": 0.0006,
74
+ "min_lr": 1e-06,
75
+ "warmup_steps": 100,
76
+ "grad_clip": 1.0,
77
+ "seed": 155979836,
78
+ "amp": true,
79
+ "num_workers": 0,
80
+ "log_every": 50,
81
+ "eval_every": 100,
82
+ "ckpt_every_min": 20.0,
83
+ "fresh_start": true,
84
+ "allow_mismatched_resume": false,
85
+ "nce_weight": 1.0,
86
+ "mse_weight": 1.0,
87
+ "nce_temperature": 0.07,
88
+ "cv_weight": 0.0,
89
+ "cv_target": 0.084,
90
+ "run_stage0": false,
91
+ "run_stage1": false,
92
+ "run_stage2": false,
93
+ "run_stage3": true,
94
+ "resume": true
95
+ },
96
+ "final": {
97
+ "mimicry_r1": 0.024000000208616257,
98
+ "cos_to_target": 0.2790161371231079,
99
+ "self_cos": 0.04439602047204971,
100
+ "erank": 12.880941310844179,
101
+ "cv": 0.4343955993918472,
102
+ "n": 5120,
103
+ "frame_r1_after_rotation": 0.026499999687075615,
104
+ "frame_cos_after_rotation": 0.3973197340965271,
105
+ "frame_n_heldout": 2000,
106
+ "aleph_amp_by_block": [
107
+ 0.00541,
108
+ 0.00658,
109
+ 0.00642,
110
+ 0.00696,
111
+ 0.00832,
112
+ 0.00789,
113
+ 0.00983,
114
+ 0.00686,
115
+ 0.01041,
116
+ 0.00878,
117
+ 0.01045,
118
+ 0.01144
119
+ ],
120
+ "aleph_w_abs_by_block": [
121
+ 0.10674,
122
+ 0.13238,
123
+ 0.13011,
124
+ 0.14359,
125
+ 0.17262,
126
+ 0.16496,
127
+ 0.20745,
128
+ 0.14398,
129
+ 0.21768,
130
+ 0.18527,
131
+ 0.22018,
132
+ 0.24169
133
+ ],
134
+ "aleph_gate_sig": [
135
+ [
136
+ 0.0503,
137
+ 0.0504,
138
+ 0.0515
139
+ ],
140
+ [
141
+ 0.0485,
142
+ 0.0515,
143
+ 0.0492
144
+ ],
145
+ [
146
+ 0.0505,
147
+ 0.0493,
148
+ 0.0486
149
+ ],
150
+ [
151
+ 0.0483,
152
+ 0.0489,
153
+ 0.0483
154
+ ],
155
+ [
156
+ 0.0481,
157
+ 0.048,
158
+ 0.0485
159
+ ],
160
+ [
161
+ 0.0479,
162
+ 0.0482,
163
+ 0.0472
164
+ ],
165
+ [
166
+ 0.0478,
167
+ 0.0469,
168
+ 0.0475
169
+ ],
170
+ [
171
+ 0.0472,
172
+ 0.0483,
173
+ 0.0475
174
+ ],
175
+ [
176
+ 0.048,
177
+ 0.0479,
178
+ 0.0469
179
+ ],
180
+ [
181
+ 0.0467,
182
+ 0.0463,
183
+ 0.0479
184
+ ],
185
+ [
186
+ 0.048,
187
+ 0.0459,
188
+ 0.0478
189
+ ],
190
+ [
191
+ 0.0472,
192
+ 0.0471,
193
+ 0.0477
194
+ ]
195
+ ],
196
+ "aleph_drift_by_block": [
197
+ 0.28697,
198
+ 0.28798,
199
+ 0.29696,
200
+ 0.2556,
201
+ 0.29396,
202
+ 0.26322,
203
+ 0.26512,
204
+ 0.21403,
205
+ 0.26556,
206
+ 0.25581,
207
+ 0.3039,
208
+ 0.33921
209
+ ],
210
+ "splat_mass_abs_mean_b00": 0.12691,
211
+ "splat_mass_abs_max_b00": 0.35513,
212
+ "splat_mass_net_mean_b00": -0.0012,
213
+ "splat_frac_neg_b00": 0.5,
214
+ "splat_mass_abs_mean_b01": 0.12579,
215
+ "splat_mass_abs_max_b01": 0.38853,
216
+ "splat_mass_net_mean_b01": -0.00023,
217
+ "splat_frac_neg_b01": 0.5547,
218
+ "splat_mass_abs_mean_b02": 0.12745,
219
+ "splat_mass_abs_max_b02": 0.4058,
220
+ "splat_mass_net_mean_b02": 0.00278,
221
+ "splat_frac_neg_b02": 0.4297,
222
+ "splat_mass_abs_mean_b03": 0.12502,
223
+ "splat_mass_abs_max_b03": 0.38124,
224
+ "splat_mass_net_mean_b03": -0.00281,
225
+ "splat_frac_neg_b03": 0.5859,
226
+ "splat_mass_abs_mean_b04": 0.12637,
227
+ "splat_mass_abs_max_b04": 0.39152,
228
+ "splat_mass_net_mean_b04": 8e-05,
229
+ "splat_frac_neg_b04": 0.5078,
230
+ "splat_mass_abs_mean_b05": 0.12798,
231
+ "splat_mass_abs_max_b05": 0.38665,
232
+ "splat_mass_net_mean_b05": -0.00148,
233
+ "splat_frac_neg_b05": 0.4766,
234
+ "splat_mass_abs_mean_b06": 0.12715,
235
+ "splat_mass_abs_max_b06": 0.39234,
236
+ "splat_mass_net_mean_b06": -0.0049,
237
+ "splat_frac_neg_b06": 0.5078,
238
+ "splat_mass_abs_mean_b07": 0.12922,
239
+ "splat_mass_abs_max_b07": 0.38137,
240
+ "splat_mass_net_mean_b07": -0.0028,
241
+ "splat_frac_neg_b07": 0.5547,
242
+ "splat_mass_abs_mean_b08": 0.12675,
243
+ "splat_mass_abs_max_b08": 0.43478,
244
+ "splat_mass_net_mean_b08": -0.00527,
245
+ "splat_frac_neg_b08": 0.4844,
246
+ "splat_mass_abs_mean_b09": 0.12941,
247
+ "splat_mass_abs_max_b09": 0.50173,
248
+ "splat_mass_net_mean_b09": 0.00563,
249
+ "splat_frac_neg_b09": 0.5547,
250
+ "splat_mass_abs_mean_b10": 0.12616,
251
+ "splat_mass_abs_max_b10": 0.4139,
252
+ "splat_mass_net_mean_b10": -0.00041,
253
+ "splat_frac_neg_b10": 0.5312,
254
+ "splat_mass_abs_mean_b11": 0.1268,
255
+ "splat_mass_abs_max_b11": 0.45663,
256
+ "splat_mass_net_mean_b11": -0.00127,
257
+ "splat_frac_neg_b11": 0.4688,
258
+ "aleph_amp_b00": 0.00541,
259
+ "aleph_amp_b01": 0.00658,
260
+ "aleph_amp_b02": 0.00642,
261
+ "aleph_amp_b03": 0.00696,
262
+ "aleph_amp_b04": 0.00832,
263
+ "aleph_amp_b05": 0.00789,
264
+ "aleph_amp_b06": 0.00983,
265
+ "aleph_amp_b07": 0.00686,
266
+ "aleph_amp_b08": 0.01041,
267
+ "aleph_amp_b09": 0.00878,
268
+ "aleph_amp_b10": 0.01045,
269
+ "aleph_amp_b11": 0.01144,
270
+ "aleph_drift_b00": 0.28697,
271
+ "aleph_drift_b01": 0.28798,
272
+ "aleph_drift_b02": 0.29696,
273
+ "aleph_drift_b03": 0.2556,
274
+ "aleph_drift_b04": 0.29396,
275
+ "aleph_drift_b05": 0.26322,
276
+ "aleph_drift_b06": 0.26512,
277
+ "aleph_drift_b07": 0.21403,
278
+ "aleph_drift_b08": 0.26556,
279
+ "aleph_drift_b09": 0.25581,
280
+ "aleph_drift_b10": 0.3039,
281
+ "aleph_drift_b11": 0.33921,
282
+ "aleph_gate_mean_b00": 0.05073333333333333,
283
+ "aleph_gate_mean_b01": 0.04973333333333333,
284
+ "aleph_gate_mean_b02": 0.049466666666666666,
285
+ "aleph_gate_mean_b03": 0.04850000000000001,
286
+ "aleph_gate_mean_b04": 0.0482,
287
+ "aleph_gate_mean_b05": 0.04776666666666666,
288
+ "aleph_gate_mean_b06": 0.0474,
289
+ "aleph_gate_mean_b07": 0.04766666666666667,
290
+ "aleph_gate_mean_b08": 0.047599999999999996,
291
+ "aleph_gate_mean_b09": 0.046966666666666664,
292
+ "aleph_gate_mean_b10": 0.04723333333333333,
293
+ "aleph_gate_mean_b11": 0.04733333333333334
294
+ }
295
+ }
runs/alephlm0-small-a4_splat-cbk-s0/checkpoints/state.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:260aaaaaeb47ca9295c30767e11cf7c3d11118386b5a8594062cb20536ae9778
3
  size 753154472
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:da56f5562a2f28bffcffdd90552639b567d3bbec1e36aabd0fd2e67de9676712
3
  size 753154472
runs/alephlm0-small-a4_splat-cbk-s0/checkpoints/tokenizer/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
runs/alephlm0-small-a4_splat-cbk-s0/checkpoints/tokenizer/tokenizer_config.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "cls_token": "[CLS]",
4
+ "do_lower_case": true,
5
+ "is_local": false,
6
+ "local_files_only": false,
7
+ "mask_token": "[MASK]",
8
+ "model_max_length": 512,
9
+ "pad_token": "[PAD]",
10
+ "sep_token": "[SEP]",
11
+ "strip_accents": null,
12
+ "tokenize_chinese_chars": true,
13
+ "tokenizer_class": "BertTokenizer",
14
+ "unk_token": "[UNK]"
15
+ }