Hugging Face's logo

AbstractPhil
/
alephlm-0

Feature Extraction
TensorBoard
English
sentence-similarity
consensus-distillation
mixture-of-experts
sparse-routing
geometric-deep-learning
amoe
research-log
Model card Files
xet
 Metrics Community
AbstractPhil commited on
Commit
4d73bcd
·
verified ·
1 Parent(s): fd68fa1

final (runs/alephlm0-small-a4_splat-hg-s0/checkpoints)

Browse files
Files changed (8) hide show
  1. runs/alephlm0-small-a4_splat-hg-s0/checkpoints/best_model.pt +1 -1
  2. runs/alephlm0-small-a4_splat-hg-s0/checkpoints/best_state.pt +2 -2
  3. runs/alephlm0-small-a4_splat-hg-s0/checkpoints/final_model.pt +3 -0
  4. runs/alephlm0-small-a4_splat-hg-s0/checkpoints/metrics.json +295 -0
  5. runs/alephlm0-small-a4_splat-hg-s0/checkpoints/model_s469.pt +3 -0
  6. runs/alephlm0-small-a4_splat-hg-s0/checkpoints/state.pt +1 -1
  7. runs/alephlm0-small-a4_splat-hg-s0/checkpoints/tokenizer/tokenizer.json +0 -0
  8. runs/alephlm0-small-a4_splat-hg-s0/checkpoints/tokenizer/tokenizer_config.json +15 -0
runs/alephlm0-small-a4_splat-hg-s0/checkpoints/best_model.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:3ab5703ad25e2443af341ddbe8d6c6d076e0359deec08c6650cffcffe19ef1bb
3
  size 284991157
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ebf13073c0f20bf76c26ac85ceebe4f77962a05ae705458c7e0dcb7048f112cb
3
  size 284991157
runs/alephlm0-small-a4_splat-hg-s0/checkpoints/best_state.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:deb93d0bc3efc57e50f45c314c12b932dd220d98fe39bec58ea46d7636dc2af3
3
- size 701646411
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:134b4e2df74de389c6139c41575c9cfcd0c834433c991ee59c9ab40cfe819a56
3
+ size 701646475
runs/alephlm0-small-a4_splat-hg-s0/checkpoints/final_model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9a3fa5f3e903ec69c4cfcfb28645d0c81d53aeee5c53106be68e71a73ce96f32
3
+ size 284994593
runs/alephlm0-small-a4_splat-hg-s0/checkpoints/metrics.json ADDED
@@ -0,0 +1,295 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "profile": "_applied",
4
+ "run_name": "alephlm0-small-a4_splat-hg-s0",
5
+ "hf_repo": "AbstractPhil/alephlm-0",
6
+ "sources": [
7
+ {
8
+ "repo": "AbstractPhil/conceptual-captions-12m-webdataset-berts",
9
+ "n_chunks": 1,
10
+ "chunk_rows": 500000,
11
+ "missing": {}
12
+ }
13
+ ],
14
+ "experts": [
15
+ "bert",
16
+ "modern",
17
+ "roberta",
18
+ "albert",
19
+ "distil"
20
+ ],
21
+ "ref_expert": "bert",
22
+ "ref_hf_name": "google-bert/bert-base-uncased",
23
+ "require_all_experts": true,
24
+ "caption_field": "caption_llava",
25
+ "caption_field_candidates": [
26
+ "caption_llava",
27
+ "caption",
28
+ "caption_llava_short"
29
+ ],
30
+ "work_dir": "E:/mirel/data/alephlm0",
31
+ "keep_expert_shards": false,
32
+ "disk_floor_gb": 25.0,
33
+ "ram_resident": true,
34
+ "preflight": true,
35
+ "artifact_repo": "AbstractPhil/captionbert-8192-v2",
36
+ "targets_repo": "AbstractPhil/captionbert-8192-v2-consensus",
37
+ "push_targets": true,
38
+ "hf_push": true,
39
+ "push_every_min": 30.0,
40
+ "keep_local_ckpts": 3,
41
+ "parity_chunk": 0,
42
+ "parity_n": 64,
43
+ "parity_min_cos": 0.999,
44
+ "fit_chunks": [
45
+ 0,
46
+ 11,
47
+ 22,
48
+ 33,
49
+ 44,
50
+ 55
51
+ ],
52
+ "fit_rows_per_chunk": 4000,
53
+ "holdout_chunks": [
54
+ 60,
55
+ 61
56
+ ],
57
+ "fit_seed": 0,
58
+ "d_model": 512,
59
+ "n_heads": 8,
60
+ "n_layers": 12,
61
+ "d_ff": 2048,
62
+ "max_len": 8192,
63
+ "output_dim": 768,
64
+ "dropout": 0.1,
65
+ "pooling": "cls",
66
+ "max_tokens": 128,
67
+ "length_bucketing": true,
68
+ "bucket_window": 64,
69
+ "grad_checkpointing": true,
70
+ "vram_probe": true,
71
+ "epochs": 2,
72
+ "batch_size": 2048,
73
+ "lr": 0.0006,
74
+ "min_lr": 1e-06,
75
+ "warmup_steps": 100,
76
+ "grad_clip": 1.0,
77
+ "seed": 155979836,
78
+ "amp": true,
79
+ "num_workers": 0,
80
+ "log_every": 50,
81
+ "eval_every": 100,
82
+ "ckpt_every_min": 20.0,
83
+ "fresh_start": true,
84
+ "allow_mismatched_resume": false,
85
+ "nce_weight": 1.0,
86
+ "mse_weight": 1.0,
87
+ "nce_temperature": 0.07,
88
+ "cv_weight": 0.0,
89
+ "cv_target": 0.084,
90
+ "run_stage0": false,
91
+ "run_stage1": false,
92
+ "run_stage2": false,
93
+ "run_stage3": true,
94
+ "resume": true
95
+ },
96
+ "final": {
97
+ "mimicry_r1": 0.006000000052154064,
98
+ "cos_to_target": 0.18246760964393616,
99
+ "self_cos": 0.08537276089191437,
100
+ "erank": 4.743998262245357,
101
+ "cv": 0.9207307410668147,
102
+ "n": 5120,
103
+ "frame_r1_after_rotation": 0.005499999970197678,
104
+ "frame_cos_after_rotation": 0.33914709091186523,
105
+ "frame_n_heldout": 2000,
106
+ "aleph_amp_by_block": [
107
+ 0.00572,
108
+ 0.00701,
109
+ 0.00823,
110
+ 0.00854,
111
+ 0.00796,
112
+ 0.00716,
113
+ 0.00557,
114
+ 0.00731,
115
+ 0.00957,
116
+ 0.00762,
117
+ 0.01149,
118
+ 0.00601
119
+ ],
120
+ "aleph_w_abs_by_block": [
121
+ 0.11743,
122
+ 0.14387,
123
+ 0.17201,
124
+ 0.17891,
125
+ 0.16699,
126
+ 0.15108,
127
+ 0.11925,
128
+ 0.15448,
129
+ 0.20152,
130
+ 0.16036,
131
+ 0.23914,
132
+ 0.12869
133
+ ],
134
+ "aleph_gate_sig": [
135
+ [
136
+ 0.0489,
137
+ 0.0483,
138
+ 0.0488
139
+ ],
140
+ [
141
+ 0.0482,
142
+ 0.0497,
143
+ 0.0488
144
+ ],
145
+ [
146
+ 0.0475,
147
+ 0.0478,
148
+ 0.0481
149
+ ],
150
+ [
151
+ 0.0481,
152
+ 0.0476,
153
+ 0.0471
154
+ ],
155
+ [
156
+ 0.047,
157
+ 0.048,
158
+ 0.0476
159
+ ],
160
+ [
161
+ 0.0474,
162
+ 0.0477,
163
+ 0.0471
164
+ ],
165
+ [
166
+ 0.0474,
167
+ 0.0464,
168
+ 0.0458
169
+ ],
170
+ [
171
+ 0.0462,
172
+ 0.0479,
173
+ 0.0463
174
+ ],
175
+ [
176
+ 0.0468,
177
+ 0.0474,
178
+ 0.0478
179
+ ],
180
+ [
181
+ 0.0466,
182
+ 0.048,
183
+ 0.0465
184
+ ],
185
+ [
186
+ 0.0486,
187
+ 0.0471,
188
+ 0.0481
189
+ ],
190
+ [
191
+ 0.0467,
192
+ 0.047,
193
+ 0.0456
194
+ ]
195
+ ],
196
+ "aleph_drift_by_block": [
197
+ 0.30534,
198
+ 0.33749,
199
+ 0.33408,
200
+ 0.30525,
201
+ 0.24303,
202
+ 0.2636,
203
+ 0.19609,
204
+ 0.23861,
205
+ 0.29104,
206
+ 0.21893,
207
+ 0.27149,
208
+ 0.23101
209
+ ],
210
+ "splat_mass_abs_mean_b00": 0.12672,
211
+ "splat_mass_abs_max_b00": 0.35655,
212
+ "splat_mass_net_mean_b00": -0.00084,
213
+ "splat_frac_neg_b00": 0.4766,
214
+ "splat_mass_abs_mean_b01": 0.12552,
215
+ "splat_mass_abs_max_b01": 0.39537,
216
+ "splat_mass_net_mean_b01": -0.00111,
217
+ "splat_frac_neg_b01": 0.5859,
218
+ "splat_mass_abs_mean_b02": 0.12629,
219
+ "splat_mass_abs_max_b02": 0.39326,
220
+ "splat_mass_net_mean_b02": 0.00513,
221
+ "splat_frac_neg_b02": 0.4531,
222
+ "splat_mass_abs_mean_b03": 0.12452,
223
+ "splat_mass_abs_max_b03": 0.38168,
224
+ "splat_mass_net_mean_b03": -0.00392,
225
+ "splat_frac_neg_b03": 0.5547,
226
+ "splat_mass_abs_mean_b04": 0.12527,
227
+ "splat_mass_abs_max_b04": 0.35962,
228
+ "splat_mass_net_mean_b04": 0.00034,
229
+ "splat_frac_neg_b04": 0.4766,
230
+ "splat_mass_abs_mean_b05": 0.12688,
231
+ "splat_mass_abs_max_b05": 0.40997,
232
+ "splat_mass_net_mean_b05": 0.00027,
233
+ "splat_frac_neg_b05": 0.5312,
234
+ "splat_mass_abs_mean_b06": 0.12667,
235
+ "splat_mass_abs_max_b06": 0.41246,
236
+ "splat_mass_net_mean_b06": -0.00017,
237
+ "splat_frac_neg_b06": 0.4219,
238
+ "splat_mass_abs_mean_b07": 0.13237,
239
+ "splat_mass_abs_max_b07": 0.63628,
240
+ "splat_mass_net_mean_b07": 0.00317,
241
+ "splat_frac_neg_b07": 0.4219,
242
+ "splat_mass_abs_mean_b08": 0.12734,
243
+ "splat_mass_abs_max_b08": 0.60775,
244
+ "splat_mass_net_mean_b08": -0.00207,
245
+ "splat_frac_neg_b08": 0.4766,
246
+ "splat_mass_abs_mean_b09": 0.12485,
247
+ "splat_mass_abs_max_b09": 0.47035,
248
+ "splat_mass_net_mean_b09": 0.00149,
249
+ "splat_frac_neg_b09": 0.4844,
250
+ "splat_mass_abs_mean_b10": 0.13038,
251
+ "splat_mass_abs_max_b10": 0.48657,
252
+ "splat_mass_net_mean_b10": 0.0018,
253
+ "splat_frac_neg_b10": 0.5312,
254
+ "splat_mass_abs_mean_b11": 0.13296,
255
+ "splat_mass_abs_max_b11": 0.52757,
256
+ "splat_mass_net_mean_b11": 0.00489,
257
+ "splat_frac_neg_b11": 0.4922,
258
+ "aleph_amp_b00": 0.00572,
259
+ "aleph_amp_b01": 0.00701,
260
+ "aleph_amp_b02": 0.00823,
261
+ "aleph_amp_b03": 0.00854,
262
+ "aleph_amp_b04": 0.00796,
263
+ "aleph_amp_b05": 0.00716,
264
+ "aleph_amp_b06": 0.00557,
265
+ "aleph_amp_b07": 0.00731,
266
+ "aleph_amp_b08": 0.00957,
267
+ "aleph_amp_b09": 0.00762,
268
+ "aleph_amp_b10": 0.01149,
269
+ "aleph_amp_b11": 0.00601,
270
+ "aleph_drift_b00": 0.30534,
271
+ "aleph_drift_b01": 0.33749,
272
+ "aleph_drift_b02": 0.33408,
273
+ "aleph_drift_b03": 0.30525,
274
+ "aleph_drift_b04": 0.24303,
275
+ "aleph_drift_b05": 0.2636,
276
+ "aleph_drift_b06": 0.19609,
277
+ "aleph_drift_b07": 0.23861,
278
+ "aleph_drift_b08": 0.29104,
279
+ "aleph_drift_b09": 0.21893,
280
+ "aleph_drift_b10": 0.27149,
281
+ "aleph_drift_b11": 0.23101,
282
+ "aleph_gate_mean_b00": 0.04866666666666667,
283
+ "aleph_gate_mean_b01": 0.0489,
284
+ "aleph_gate_mean_b02": 0.0478,
285
+ "aleph_gate_mean_b03": 0.0476,
286
+ "aleph_gate_mean_b04": 0.04753333333333334,
287
+ "aleph_gate_mean_b05": 0.0474,
288
+ "aleph_gate_mean_b06": 0.046533333333333336,
289
+ "aleph_gate_mean_b07": 0.0468,
290
+ "aleph_gate_mean_b08": 0.04733333333333334,
291
+ "aleph_gate_mean_b09": 0.04703333333333334,
292
+ "aleph_gate_mean_b10": 0.04793333333333333,
293
+ "aleph_gate_mean_b11": 0.04643333333333333
294
+ }
295
+ }
runs/alephlm0-small-a4_splat-hg-s0/checkpoints/model_s469.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4bce4cca5aaf12d2c4e06be085ab55cfa48e40d764240b323cc5d08b2991156a
3
+ size 284991157
runs/alephlm0-small-a4_splat-hg-s0/checkpoints/state.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:8a9cf1103d5393a3a802c8be16e39a119d02a363f1c797f35b0d5b6226d78b4b
3
  size 701560408
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:908703521315a14eb659fd3a2eea493547c93729f87fb2d0591d13a1c68b4f05
3
  size 701560408
runs/alephlm0-small-a4_splat-hg-s0/checkpoints/tokenizer/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
runs/alephlm0-small-a4_splat-hg-s0/checkpoints/tokenizer/tokenizer_config.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "cls_token": "[CLS]",
4
+ "do_lower_case": true,
5
+ "is_local": false,
6
+ "local_files_only": false,
7
+ "mask_token": "[MASK]",
8
+ "model_max_length": 512,
9
+ "pad_token": "[PAD]",
10
+ "sep_token": "[SEP]",
11
+ "strip_accents": null,
12
+ "tokenize_chinese_chars": true,
13
+ "tokenizer_class": "BertTokenizer",
14
+ "unk_token": "[UNK]"
15
+ }