Hugging Face's logo

AbstractPhil
/
alephlm-0

Feature Extraction
TensorBoard
English
sentence-similarity
consensus-distillation
mixture-of-experts
sparse-routing
geometric-deep-learning
amoe
research-log
Model card Files
xet
 Metrics Community
AbstractPhil commited on
Commit
b31ddda
·
verified ·
1 Parent(s): 7730210

final (runs/alephlm0-small-a4_splat-cbk-rot-wa-s1-e6/checkpoints)

Browse files
Files changed (8) hide show
  1. runs/alephlm0-small-a4_splat-cbk-rot-wa-s1-e6/checkpoints/best_model.pt +1 -1
  2. runs/alephlm0-small-a4_splat-cbk-rot-wa-s1-e6/checkpoints/best_state.pt +1 -1
  3. runs/alephlm0-small-a4_splat-cbk-rot-wa-s1-e6/checkpoints/final_model.pt +3 -0
  4. runs/alephlm0-small-a4_splat-cbk-rot-wa-s1-e6/checkpoints/metrics.json +295 -0
  5. runs/alephlm0-small-a4_splat-cbk-rot-wa-s1-e6/checkpoints/model_s1412.pt +3 -0
  6. runs/alephlm0-small-a4_splat-cbk-rot-wa-s1-e6/checkpoints/state.pt +1 -1
  7. runs/alephlm0-small-a4_splat-cbk-rot-wa-s1-e6/checkpoints/tokenizer/tokenizer.json +0 -0
  8. runs/alephlm0-small-a4_splat-cbk-rot-wa-s1-e6/checkpoints/tokenizer/tokenizer_config.json +15 -0
runs/alephlm0-small-a4_splat-cbk-rot-wa-s1-e6/checkpoints/best_model.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:54ebcf8f24397ef43e717b169c132465efb8ca913a18a63affe9058911d7d27d
3
  size 297568629
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c3a72d23ebda61c9b699b4c47225908826b33660a558303e85afd5a927dc98fd
3
  size 297568629
runs/alephlm0-small-a4_splat-cbk-rot-wa-s1-e6/checkpoints/best_state.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:357bdca8bae9fa5250067ff7c88e77e77d7fb116a968c7fd089020bea2b05daa
3
  size 791027019
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:38a22208493ee5983cb88c294b9ac2851ef35a3cdb383a57282646bb21992929
3
  size 791027019
runs/alephlm0-small-a4_splat-cbk-rot-wa-s1-e6/checkpoints/final_model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3a920d57a5962ca052dc42dc8daea2a32f8b9401b90803c453f986ad82cd6a8b
3
+ size 297572065
runs/alephlm0-small-a4_splat-cbk-rot-wa-s1-e6/checkpoints/metrics.json ADDED
@@ -0,0 +1,295 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "profile": "_applied",
4
+ "run_name": "alephlm0-small-a4_splat-cbk-rot-wa-s1-e6",
5
+ "hf_repo": "AbstractPhil/alephlm-0",
6
+ "sources": [
7
+ {
8
+ "repo": "AbstractPhil/conceptual-captions-12m-webdataset-berts",
9
+ "n_chunks": 1,
10
+ "chunk_rows": 500000,
11
+ "missing": {}
12
+ }
13
+ ],
14
+ "experts": [
15
+ "bert",
16
+ "modern",
17
+ "roberta",
18
+ "albert",
19
+ "distil"
20
+ ],
21
+ "ref_expert": "bert",
22
+ "ref_hf_name": "google-bert/bert-base-uncased",
23
+ "require_all_experts": true,
24
+ "caption_field": "caption_llava",
25
+ "caption_field_candidates": [
26
+ "caption_llava",
27
+ "caption",
28
+ "caption_llava_short"
29
+ ],
30
+ "work_dir": "E:/mirel/data/alephlm0",
31
+ "keep_expert_shards": false,
32
+ "disk_floor_gb": 25.0,
33
+ "ram_resident": true,
34
+ "preflight": true,
35
+ "artifact_repo": "AbstractPhil/captionbert-8192-v2",
36
+ "targets_repo": "AbstractPhil/captionbert-8192-v2-consensus",
37
+ "push_targets": true,
38
+ "hf_push": true,
39
+ "push_every_min": 30.0,
40
+ "keep_local_ckpts": 3,
41
+ "parity_chunk": 0,
42
+ "parity_n": 64,
43
+ "parity_min_cos": 0.999,
44
+ "fit_chunks": [
45
+ 0,
46
+ 11,
47
+ 22,
48
+ 33,
49
+ 44,
50
+ 55
51
+ ],
52
+ "fit_rows_per_chunk": 4000,
53
+ "holdout_chunks": [
54
+ 60,
55
+ 61
56
+ ],
57
+ "fit_seed": 0,
58
+ "d_model": 512,
59
+ "n_heads": 8,
60
+ "n_layers": 12,
61
+ "d_ff": 2048,
62
+ "max_len": 8192,
63
+ "output_dim": 768,
64
+ "dropout": 0.1,
65
+ "pooling": "cls",
66
+ "max_tokens": 128,
67
+ "length_bucketing": true,
68
+ "bucket_window": 64,
69
+ "grad_checkpointing": true,
70
+ "vram_probe": true,
71
+ "epochs": 6,
72
+ "batch_size": 2048,
73
+ "lr": 0.0006,
74
+ "min_lr": 1e-06,
75
+ "warmup_steps": 100,
76
+ "grad_clip": 1.0,
77
+ "seed": 46212409,
78
+ "amp": true,
79
+ "num_workers": 0,
80
+ "log_every": 50,
81
+ "eval_every": 100,
82
+ "ckpt_every_min": 20.0,
83
+ "fresh_start": true,
84
+ "allow_mismatched_resume": false,
85
+ "nce_weight": 1.0,
86
+ "mse_weight": 1.0,
87
+ "nce_temperature": 0.07,
88
+ "cv_weight": 0.0,
89
+ "cv_target": 0.084,
90
+ "run_stage0": false,
91
+ "run_stage1": false,
92
+ "run_stage2": false,
93
+ "run_stage3": true,
94
+ "resume": true
95
+ },
96
+ "final": {
97
+ "mimicry_r1": 0.8240000009536743,
98
+ "cos_to_target": 0.6286168694496155,
99
+ "self_cos": 0.003870457410812378,
100
+ "erank": 38.02185952298474,
101
+ "cv": 0.14242844717611117,
102
+ "n": 5120,
103
+ "frame_r1_after_rotation": 0.8529999852180481,
104
+ "frame_cos_after_rotation": 0.6412888169288635,
105
+ "frame_n_heldout": 2000,
106
+ "aleph_amp_by_block": [
107
+ 0.0064,
108
+ 0.00808,
109
+ 0.00862,
110
+ 0.00943,
111
+ 0.0093,
112
+ 0.00924,
113
+ 0.00934,
114
+ 0.00869,
115
+ 0.00979,
116
+ 0.00977,
117
+ 0.00968,
118
+ 0.01024
119
+ ],
120
+ "aleph_w_abs_by_block": [
121
+ 0.12232,
122
+ 0.1614,
123
+ 0.17613,
124
+ 0.18879,
125
+ 0.18125,
126
+ 0.18654,
127
+ 0.18383,
128
+ 0.17379,
129
+ 0.19374,
130
+ 0.18633,
131
+ 0.19418,
132
+ 0.19934
133
+ ],
134
+ "aleph_gate_sig": [
135
+ [
136
+ 0.054,
137
+ 0.0486,
138
+ 0.0557
139
+ ],
140
+ [
141
+ 0.0546,
142
+ 0.0487,
143
+ 0.0484
144
+ ],
145
+ [
146
+ 0.0481,
147
+ 0.0498,
148
+ 0.049
149
+ ],
150
+ [
151
+ 0.0479,
152
+ 0.0508,
153
+ 0.0511
154
+ ],
155
+ [
156
+ 0.0522,
157
+ 0.0521,
158
+ 0.0498
159
+ ],
160
+ [
161
+ 0.0481,
162
+ 0.0508,
163
+ 0.049
164
+ ],
165
+ [
166
+ 0.0516,
167
+ 0.0506,
168
+ 0.0503
169
+ ],
170
+ [
171
+ 0.0508,
172
+ 0.0493,
173
+ 0.0497
174
+ ],
175
+ [
176
+ 0.0528,
177
+ 0.0491,
178
+ 0.05
179
+ ],
180
+ [
181
+ 0.0547,
182
+ 0.0497,
183
+ 0.0522
184
+ ],
185
+ [
186
+ 0.0514,
187
+ 0.0481,
188
+ 0.0502
189
+ ],
190
+ [
191
+ 0.0514,
192
+ 0.0509,
193
+ 0.0518
194
+ ]
195
+ ],
196
+ "aleph_drift_by_block": [
197
+ 0.44777,
198
+ 0.42475,
199
+ 0.33748,
200
+ 0.43117,
201
+ 0.40789,
202
+ 0.34699,
203
+ 0.38819,
204
+ 0.38825,
205
+ 0.54836,
206
+ 0.60258,
207
+ 0.50937,
208
+ 0.60661
209
+ ],
210
+ "splat_mass_abs_mean_b00": 0.13188,
211
+ "splat_mass_abs_max_b00": 0.14945,
212
+ "splat_mass_net_mean_b00": 7e-05,
213
+ "splat_frac_neg_b00": 0.4922,
214
+ "splat_mass_abs_mean_b01": 0.13115,
215
+ "splat_mass_abs_max_b01": 0.15119,
216
+ "splat_mass_net_mean_b01": 0.00153,
217
+ "splat_frac_neg_b01": 0.4453,
218
+ "splat_mass_abs_mean_b02": 0.13222,
219
+ "splat_mass_abs_max_b02": 0.1636,
220
+ "splat_mass_net_mean_b02": -0.00021,
221
+ "splat_frac_neg_b02": 0.4609,
222
+ "splat_mass_abs_mean_b03": 0.13198,
223
+ "splat_mass_abs_max_b03": 0.16875,
224
+ "splat_mass_net_mean_b03": -0.00107,
225
+ "splat_frac_neg_b03": 0.5391,
226
+ "splat_mass_abs_mean_b04": 0.13185,
227
+ "splat_mass_abs_max_b04": 0.16273,
228
+ "splat_mass_net_mean_b04": 0.0002,
229
+ "splat_frac_neg_b04": 0.5,
230
+ "splat_mass_abs_mean_b05": 0.1326,
231
+ "splat_mass_abs_max_b05": 0.16937,
232
+ "splat_mass_net_mean_b05": -0.00018,
233
+ "splat_frac_neg_b05": 0.4609,
234
+ "splat_mass_abs_mean_b06": 0.13216,
235
+ "splat_mass_abs_max_b06": 0.15894,
236
+ "splat_mass_net_mean_b06": -0.00139,
237
+ "splat_frac_neg_b06": 0.5391,
238
+ "splat_mass_abs_mean_b07": 0.13343,
239
+ "splat_mass_abs_max_b07": 0.1611,
240
+ "splat_mass_net_mean_b07": 0.00371,
241
+ "splat_frac_neg_b07": 0.4609,
242
+ "splat_mass_abs_mean_b08": 0.13341,
243
+ "splat_mass_abs_max_b08": 0.18853,
244
+ "splat_mass_net_mean_b08": -0.00378,
245
+ "splat_frac_neg_b08": 0.5703,
246
+ "splat_mass_abs_mean_b09": 0.13395,
247
+ "splat_mass_abs_max_b09": 0.1673,
248
+ "splat_mass_net_mean_b09": 0.00145,
249
+ "splat_frac_neg_b09": 0.4609,
250
+ "splat_mass_abs_mean_b10": 0.1359,
251
+ "splat_mass_abs_max_b10": 0.18227,
252
+ "splat_mass_net_mean_b10": 0.00352,
253
+ "splat_frac_neg_b10": 0.4531,
254
+ "splat_mass_abs_mean_b11": 0.13068,
255
+ "splat_mass_abs_max_b11": 0.16189,
256
+ "splat_mass_net_mean_b11": 0.00168,
257
+ "splat_frac_neg_b11": 0.5,
258
+ "aleph_amp_b00": 0.0064,
259
+ "aleph_amp_b01": 0.00808,
260
+ "aleph_amp_b02": 0.00862,
261
+ "aleph_amp_b03": 0.00943,
262
+ "aleph_amp_b04": 0.0093,
263
+ "aleph_amp_b05": 0.00924,
264
+ "aleph_amp_b06": 0.00934,
265
+ "aleph_amp_b07": 0.00869,
266
+ "aleph_amp_b08": 0.00979,
267
+ "aleph_amp_b09": 0.00977,
268
+ "aleph_amp_b10": 0.00968,
269
+ "aleph_amp_b11": 0.01024,
270
+ "aleph_drift_b00": 0.44777,
271
+ "aleph_drift_b01": 0.42475,
272
+ "aleph_drift_b02": 0.33748,
273
+ "aleph_drift_b03": 0.43117,
274
+ "aleph_drift_b04": 0.40789,
275
+ "aleph_drift_b05": 0.34699,
276
+ "aleph_drift_b06": 0.38819,
277
+ "aleph_drift_b07": 0.38825,
278
+ "aleph_drift_b08": 0.54836,
279
+ "aleph_drift_b09": 0.60258,
280
+ "aleph_drift_b10": 0.50937,
281
+ "aleph_drift_b11": 0.60661,
282
+ "aleph_gate_mean_b00": 0.05276666666666666,
283
+ "aleph_gate_mean_b01": 0.05056666666666667,
284
+ "aleph_gate_mean_b02": 0.048966666666666665,
285
+ "aleph_gate_mean_b03": 0.04993333333333333,
286
+ "aleph_gate_mean_b04": 0.05136666666666667,
287
+ "aleph_gate_mean_b05": 0.049300000000000004,
288
+ "aleph_gate_mean_b06": 0.050833333333333335,
289
+ "aleph_gate_mean_b07": 0.04993333333333333,
290
+ "aleph_gate_mean_b08": 0.050633333333333336,
291
+ "aleph_gate_mean_b09": 0.0522,
292
+ "aleph_gate_mean_b10": 0.0499,
293
+ "aleph_gate_mean_b11": 0.05136666666666667
294
+ }
295
+ }
runs/alephlm0-small-a4_splat-cbk-rot-wa-s1-e6/checkpoints/model_s1412.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd5320dfeb50d119f22ee6f70438de34486825eda476d183a28f00ac23ac301e
3
+ size 297572065
runs/alephlm0-small-a4_splat-cbk-rot-wa-s1-e6/checkpoints/state.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:3f398de0b07bd565deaf746aab2b919b5b872d3a3bdea250431f7f3bcee3eca0
3
  size 790917976
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:84bdec746838dd6a0a212e7de10012cdf10d30cfa067c14e90e8a67a99e3170f
3
  size 790917976
runs/alephlm0-small-a4_splat-cbk-rot-wa-s1-e6/checkpoints/tokenizer/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
runs/alephlm0-small-a4_splat-cbk-rot-wa-s1-e6/checkpoints/tokenizer/tokenizer_config.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "cls_token": "[CLS]",
4
+ "do_lower_case": true,
5
+ "is_local": false,
6
+ "local_files_only": false,
7
+ "mask_token": "[MASK]",
8
+ "model_max_length": 512,
9
+ "pad_token": "[PAD]",
10
+ "sep_token": "[SEP]",
11
+ "strip_accents": null,
12
+ "tokenize_chinese_chars": true,
13
+ "tokenizer_class": "BertTokenizer",
14
+ "unk_token": "[UNK]"
15
+ }