scombat commited on
Commit
f0686c7
·
verified ·
1 Parent(s): a066a36

Add v7_small_l6-fp16 (six-source development model)

Browse files
Files changed (10) hide show
  1. .gitattributes +1 -0
  2. README.md +52 -0
  3. config.json +108 -0
  4. eval.json +600 -0
  5. footprint.json +1 -0
  6. model.safetensors +3 -0
  7. tokenizer.json +3 -0
  8. tokenizer_config.json +29 -0
  9. variant.json +6 -0
  10. vocab_map.npy +3 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ pipeline_tag: text-classification
4
+ base_model: jhu-clsp/mmBERT-small
5
+ base_model_relation: finetune
6
+ inference: false
7
+ tags:
8
+ - routing
9
+ - llm-routing
10
+ - prompt-classification
11
+ - modernbert
12
+ - multilingual
13
+ - private-dev
14
+ ---
15
+
16
+ # s1d1-router, six sources, tiny, PyTorch fp16 (safetensors)
17
+
18
+ > **Private development model. Do not publish.** Trained on six sources, including lmsys-chat-1m and Magpie. The
19
+ > publishable version will be retrained on four sources only (WildChat, UltraChat, Aya, OASST2). License not decided.
20
+
21
+ A "System 1" router: it reads a user prompt and returns its complexity score (0.01 to 1.00), its task type (19 classes)
22
+ and whether it needs tools. It never picks a model; the deployer maps these outputs to models with rules.
23
+
24
+ - Variant: `v7_small_l6-fp16`, mmBERT-small cut to 6 layers, 32k vocabulary, distilled from the full model.
25
+ - Format: PyTorch fp16 (safetensors), 85 MB.
26
+ - Outputs: 21 logits. Output 0 is the complexity in tier units (score = 0.5 + 0.25 x output, clipped to 0.01..1.00),
27
+ outputs 1 to 19 are the task logits, output 20 is the needs_tools logit. Task order and decoding:
28
+ `release/s1d1-router/s1d1_router.py` in the project.
29
+ - Pruned vocabulary variants ship `vocab_map.npy` (full tokenizer id -> pruned id); the project's loader applies it.
30
+
31
+ ## Measures (validation, 4,914 rows, six sources, unweighted)
32
+
33
+ | complexity QWK | complexity MAE | task accuracy | task macro-F1 | needs_tools F1 | needs_tools AUROC |
34
+ |---|---|---|---|---|---|
35
+ | 0.649 | 0.091 | 0.782 | 0.662 | 0.484 | 0.904 |
36
+
37
+ Agreement with the full fp32 model on validation: same complexity tier 79.6%, same task
38
+ 81.3%, same needs_tools 96.7%.
39
+ Latency, batch 1, transformers backend on an Apple M4 Max: p50 3.1 ms, p95 8.0 ms.
40
+ Full results and method: `docs/15-catalog-v4-v7.md` in the project; `eval.json` in this repo.
41
+
42
+ ## Use
43
+
44
+ ```bash
45
+ hf download scombat/s1d1-router-6src-tiny --local-dir s1d1-router-tiny
46
+ ```
47
+
48
+ ```python
49
+ from s1d1_router import Router # release/s1d1-router/s1d1_router.py in the project
50
+ router = Router("s1d1-router-tiny")
51
+ router.route(["Quelle est la capitale de l'Australie ?"])
52
+ ```
config.json ADDED
@@ -0,0 +1,108 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "ModernBertForSequenceClassification"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 2,
8
+ "classifier_activation": "gelu",
9
+ "classifier_bias": false,
10
+ "classifier_dropout": 0.0,
11
+ "classifier_pooling": "mean",
12
+ "cls_token_id": 1,
13
+ "decoder_bias": true,
14
+ "deterministic_flash_attn": false,
15
+ "dtype": "float16",
16
+ "embedding_dropout": 0.0,
17
+ "eos_token_id": 1,
18
+ "global_attn_every_n_layers": 3,
19
+ "gradient_checkpointing": false,
20
+ "hidden_activation": "gelu",
21
+ "hidden_size": 384,
22
+ "id2label": {
23
+ "0": "LABEL_0",
24
+ "1": "LABEL_1",
25
+ "2": "LABEL_2",
26
+ "3": "LABEL_3",
27
+ "4": "LABEL_4",
28
+ "5": "LABEL_5",
29
+ "6": "LABEL_6",
30
+ "7": "LABEL_7",
31
+ "8": "LABEL_8",
32
+ "9": "LABEL_9",
33
+ "10": "LABEL_10",
34
+ "11": "LABEL_11",
35
+ "12": "LABEL_12",
36
+ "13": "LABEL_13",
37
+ "14": "LABEL_14",
38
+ "15": "LABEL_15",
39
+ "16": "LABEL_16",
40
+ "17": "LABEL_17",
41
+ "18": "LABEL_18",
42
+ "19": "LABEL_19",
43
+ "20": "LABEL_20"
44
+ },
45
+ "initializer_cutoff_factor": 2.0,
46
+ "initializer_range": 0.02,
47
+ "intermediate_size": 1152,
48
+ "label2id": {
49
+ "LABEL_0": 0,
50
+ "LABEL_1": 1,
51
+ "LABEL_10": 10,
52
+ "LABEL_11": 11,
53
+ "LABEL_12": 12,
54
+ "LABEL_13": 13,
55
+ "LABEL_14": 14,
56
+ "LABEL_15": 15,
57
+ "LABEL_16": 16,
58
+ "LABEL_17": 17,
59
+ "LABEL_18": 18,
60
+ "LABEL_19": 19,
61
+ "LABEL_2": 2,
62
+ "LABEL_20": 20,
63
+ "LABEL_3": 3,
64
+ "LABEL_4": 4,
65
+ "LABEL_5": 5,
66
+ "LABEL_6": 6,
67
+ "LABEL_7": 7,
68
+ "LABEL_8": 8,
69
+ "LABEL_9": 9
70
+ },
71
+ "layer_norm_eps": 1e-05,
72
+ "layer_types": [
73
+ "full_attention",
74
+ "sliding_attention",
75
+ "sliding_attention",
76
+ "full_attention",
77
+ "sliding_attention",
78
+ "sliding_attention"
79
+ ],
80
+ "local_attention": 128,
81
+ "mask_token_id": 4,
82
+ "max_position_embeddings": 8192,
83
+ "mlp_bias": false,
84
+ "mlp_dropout": 0.0,
85
+ "model_type": "modernbert",
86
+ "norm_bias": false,
87
+ "norm_eps": 1e-05,
88
+ "num_attention_heads": 6,
89
+ "num_hidden_layers": 6,
90
+ "pad_token_id": 0,
91
+ "position_embedding_type": "sans_pos",
92
+ "rope_parameters": {
93
+ "full_attention": {
94
+ "rope_theta": 160000,
95
+ "rope_type": "default"
96
+ },
97
+ "sliding_attention": {
98
+ "rope_theta": 160000,
99
+ "rope_type": "default"
100
+ }
101
+ },
102
+ "sep_token_id": 1,
103
+ "sparse_pred_ignore_index": -100,
104
+ "sparse_prediction": false,
105
+ "tie_word_embeddings": true,
106
+ "transformers_version": "5.19.0",
107
+ "vocab_size": 32253
108
+ }
eval.json ADDED
@@ -0,0 +1,600 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "variant": "v7_small_l6-fp16",
3
+ "backend": "torch",
4
+ "size_bytes": 84514110,
5
+ "validation_all": {
6
+ "unweighted": {
7
+ "rows": 4914,
8
+ "weighted": false,
9
+ "complexity": {
10
+ "rows": 4850,
11
+ "mae": 0.09123653288428316,
12
+ "spearman": 0.6593453452413092,
13
+ "qwk_rounded": 0.6490167052545179,
14
+ "confusion_true_by_pred": [
15
+ [
16
+ 92.0,
17
+ 198.0,
18
+ 18.0,
19
+ 0.0,
20
+ 0.0
21
+ ],
22
+ [
23
+ 48.0,
24
+ 2451.0,
25
+ 377.0,
26
+ 2.0,
27
+ 0.0
28
+ ],
29
+ [
30
+ 5.0,
31
+ 463.0,
32
+ 842.0,
33
+ 35.0,
34
+ 0.0
35
+ ],
36
+ [
37
+ 0.0,
38
+ 22.0,
39
+ 125.0,
40
+ 164.0,
41
+ 0.0
42
+ ],
43
+ [
44
+ 0.0,
45
+ 1.0,
46
+ 4.0,
47
+ 3.0,
48
+ 0.0
49
+ ]
50
+ ],
51
+ "support": {
52
+ "trivial": 308,
53
+ "low": 2878,
54
+ "medium": 1345,
55
+ "high": 311,
56
+ "expert": 8
57
+ }
58
+ },
59
+ "task": {
60
+ "accuracy": 0.7816442816442817,
61
+ "macro_f1": 0.6623087453835308,
62
+ "macro_f1_present": 0.6623087453835308,
63
+ "per_class": {
64
+ "information_seeking": {
65
+ "f1": 0.8244226510597912,
66
+ "support": 1528
67
+ },
68
+ "reasoning": {
69
+ "f1": 0.5149700598802395,
70
+ "support": 92
71
+ },
72
+ "planning": {
73
+ "f1": 0.6474820143884892,
74
+ "support": 81
75
+ },
76
+ "editing": {
77
+ "f1": 0.7917737789203085,
78
+ "support": 193
79
+ },
80
+ "summarization": {
81
+ "f1": 0.7649402390438247,
82
+ "support": 128
83
+ },
84
+ "extraction": {
85
+ "f1": 0.7352297592997812,
86
+ "support": 229
87
+ },
88
+ "classification": {
89
+ "f1": 0.7932960893854749,
90
+ "support": 89
91
+ },
92
+ "translation": {
93
+ "f1": 0.9041095890410958,
94
+ "support": 73
95
+ },
96
+ "coding_debugging": {
97
+ "f1": 0.8478260869565217,
98
+ "support": 453
99
+ },
100
+ "math": {
101
+ "f1": 0.8530020703933747,
102
+ "support": 236
103
+ },
104
+ "data_analysis": {
105
+ "f1": 0.5882352941176471,
106
+ "support": 22
107
+ },
108
+ "writing": {
109
+ "f1": 0.8550148957298908,
110
+ "support": 991
111
+ },
112
+ "role_playing": {
113
+ "f1": 0.7544910179640718,
114
+ "support": 83
115
+ },
116
+ "advice_seeking": {
117
+ "f1": 0.6099290780141844,
118
+ "support": 157
119
+ },
120
+ "brainstorming": {
121
+ "f1": 0.5281385281385281,
122
+ "support": 134
123
+ },
124
+ "completion": {
125
+ "f1": 0.6311787072243346,
126
+ "support": 267
127
+ },
128
+ "action_execution": {
129
+ "f1": 0.25,
130
+ "support": 11
131
+ },
132
+ "chat": {
133
+ "f1": 0.41025641025641024,
134
+ "support": 91
135
+ },
136
+ "other": {
137
+ "f1": 0.27956989247311825,
138
+ "support": 56
139
+ }
140
+ }
141
+ },
142
+ "needs_tools": {
143
+ "f1": 0.4841075794621027,
144
+ "auroc": 0.9042765866209262,
145
+ "positives": 250
146
+ }
147
+ },
148
+ "weighted": {
149
+ "rows": 4914,
150
+ "weighted": true,
151
+ "complexity": {
152
+ "rows": 4850,
153
+ "mae": 0.08725421027348824,
154
+ "spearman": 0.6160875172388919,
155
+ "qwk_rounded": 0.596045096988439,
156
+ "confusion_true_by_pred": [
157
+ [
158
+ 86.2,
159
+ 195.8,
160
+ 16.3,
161
+ 0.0,
162
+ 0.0
163
+ ],
164
+ [
165
+ 51.3,
166
+ 2634.9,
167
+ 406.4,
168
+ 1.7,
169
+ 0.0
170
+ ],
171
+ [
172
+ 5.2,
173
+ 467.1,
174
+ 845.1,
175
+ 34.0,
176
+ 0.0
177
+ ],
178
+ [
179
+ 0.0,
180
+ 9.3,
181
+ 52.7,
182
+ 69.4,
183
+ 0.0
184
+ ],
185
+ [
186
+ 0.0,
187
+ 0.1,
188
+ 0.5,
189
+ 0.4,
190
+ 0.0
191
+ ]
192
+ ],
193
+ "support": {
194
+ "trivial": 308,
195
+ "low": 2878,
196
+ "medium": 1345,
197
+ "high": 311,
198
+ "expert": 8
199
+ }
200
+ },
201
+ "task": {
202
+ "accuracy": 0.7848718360585144,
203
+ "macro_f1": 0.6535855109314881,
204
+ "macro_f1_present": 0.6535855109314881,
205
+ "per_class": {
206
+ "information_seeking": {
207
+ "f1": 0.832598914469181,
208
+ "support": 1528
209
+ },
210
+ "reasoning": {
211
+ "f1": 0.5297883989506079,
212
+ "support": 92
213
+ },
214
+ "planning": {
215
+ "f1": 0.6323352636377239,
216
+ "support": 81
217
+ },
218
+ "editing": {
219
+ "f1": 0.7972908310414097,
220
+ "support": 193
221
+ },
222
+ "summarization": {
223
+ "f1": 0.771281034042985,
224
+ "support": 128
225
+ },
226
+ "extraction": {
227
+ "f1": 0.739535897050014,
228
+ "support": 229
229
+ },
230
+ "classification": {
231
+ "f1": 0.7906490364919273,
232
+ "support": 89
233
+ },
234
+ "translation": {
235
+ "f1": 0.9039649545580544,
236
+ "support": 73
237
+ },
238
+ "coding_debugging": {
239
+ "f1": 0.850302929061513,
240
+ "support": 453
241
+ },
242
+ "math": {
243
+ "f1": 0.8457726621104891,
244
+ "support": 236
245
+ },
246
+ "data_analysis": {
247
+ "f1": 0.5085499923284106,
248
+ "support": 22
249
+ },
250
+ "writing": {
251
+ "f1": 0.8495558798328425,
252
+ "support": 991
253
+ },
254
+ "role_playing": {
255
+ "f1": 0.722487088007609,
256
+ "support": 83
257
+ },
258
+ "advice_seeking": {
259
+ "f1": 0.6228240602260838,
260
+ "support": 157
261
+ },
262
+ "brainstorming": {
263
+ "f1": 0.5321998740461651,
264
+ "support": 134
265
+ },
266
+ "completion": {
267
+ "f1": 0.6401729708095073,
268
+ "support": 267
269
+ },
270
+ "action_execution": {
271
+ "f1": 0.17852667118303162,
272
+ "support": 11
273
+ },
274
+ "chat": {
275
+ "f1": 0.41431580139190316,
276
+ "support": 91
277
+ },
278
+ "other": {
279
+ "f1": 0.2559724484588159,
280
+ "support": 56
281
+ }
282
+ }
283
+ },
284
+ "needs_tools": {
285
+ "f1": 0.4833044372624333,
286
+ "auroc": 0.9023762363440311,
287
+ "positives": 250
288
+ }
289
+ }
290
+ },
291
+ "validation_four_sources": {
292
+ "unweighted": {
293
+ "rows": 3130,
294
+ "weighted": false,
295
+ "complexity": {
296
+ "rows": 3101,
297
+ "mae": 0.08777755751143883,
298
+ "spearman": 0.6784958198543671,
299
+ "qwk_rounded": 0.6711515464869717,
300
+ "confusion_true_by_pred": [
301
+ [
302
+ 63.0,
303
+ 129.0,
304
+ 9.0,
305
+ 0.0,
306
+ 0.0
307
+ ],
308
+ [
309
+ 34.0,
310
+ 1567.0,
311
+ 235.0,
312
+ 2.0,
313
+ 0.0
314
+ ],
315
+ [
316
+ 2.0,
317
+ 275.0,
318
+ 566.0,
319
+ 29.0,
320
+ 0.0
321
+ ],
322
+ [
323
+ 0.0,
324
+ 8.0,
325
+ 81.0,
326
+ 100.0,
327
+ 0.0
328
+ ],
329
+ [
330
+ 0.0,
331
+ 0.0,
332
+ 0.0,
333
+ 1.0,
334
+ 0.0
335
+ ]
336
+ ],
337
+ "support": {
338
+ "trivial": 201,
339
+ "low": 1838,
340
+ "medium": 872,
341
+ "high": 189,
342
+ "expert": 1
343
+ }
344
+ },
345
+ "task": {
346
+ "accuracy": 0.7955271565495208,
347
+ "macro_f1": 0.6418206886744169,
348
+ "macro_f1_present": 0.6418206886744169,
349
+ "per_class": {
350
+ "information_seeking": {
351
+ "f1": 0.8394724874943156,
352
+ "support": 1068
353
+ },
354
+ "reasoning": {
355
+ "f1": 0.4943820224719101,
356
+ "support": 48
357
+ },
358
+ "planning": {
359
+ "f1": 0.6067415730337079,
360
+ "support": 53
361
+ },
362
+ "editing": {
363
+ "f1": 0.8,
364
+ "support": 139
365
+ },
366
+ "summarization": {
367
+ "f1": 0.78125,
368
+ "support": 98
369
+ },
370
+ "extraction": {
371
+ "f1": 0.7688022284122563,
372
+ "support": 180
373
+ },
374
+ "classification": {
375
+ "f1": 0.7578947368421053,
376
+ "support": 47
377
+ },
378
+ "translation": {
379
+ "f1": 0.907563025210084,
380
+ "support": 61
381
+ },
382
+ "coding_debugging": {
383
+ "f1": 0.8577235772357723,
384
+ "support": 246
385
+ },
386
+ "math": {
387
+ "f1": 0.7058823529411765,
388
+ "support": 41
389
+ },
390
+ "data_analysis": {
391
+ "f1": 0.18181818181818182,
392
+ "support": 5
393
+ },
394
+ "writing": {
395
+ "f1": 0.8631284916201117,
396
+ "support": 705
397
+ },
398
+ "role_playing": {
399
+ "f1": 0.7419354838709677,
400
+ "support": 30
401
+ },
402
+ "advice_seeking": {
403
+ "f1": 0.576,
404
+ "support": 75
405
+ },
406
+ "brainstorming": {
407
+ "f1": 0.5285714285714286,
408
+ "support": 83
409
+ },
410
+ "completion": {
411
+ "f1": 0.6923076923076923,
412
+ "support": 162
413
+ },
414
+ "action_execution": {
415
+ "f1": 0.3076923076923077,
416
+ "support": 8
417
+ },
418
+ "chat": {
419
+ "f1": 0.4444444444444444,
420
+ "support": 47
421
+ },
422
+ "other": {
423
+ "f1": 0.3389830508474576,
424
+ "support": 34
425
+ }
426
+ }
427
+ },
428
+ "needs_tools": {
429
+ "f1": 0.43824701195219123,
430
+ "auroc": 0.9143597647322281,
431
+ "positives": 153
432
+ }
433
+ },
434
+ "weighted": {
435
+ "rows": 3130,
436
+ "weighted": true,
437
+ "complexity": {
438
+ "rows": 3101,
439
+ "mae": 0.08461350760287671,
440
+ "spearman": 0.636775261257106,
441
+ "qwk_rounded": 0.61930401674141,
442
+ "confusion_true_by_pred": [
443
+ [
444
+ 59.1,
445
+ 128.4,
446
+ 8.1,
447
+ 0.0,
448
+ 0.0
449
+ ],
450
+ [
451
+ 36.9,
452
+ 1696.7,
453
+ 256.0,
454
+ 1.7,
455
+ 0.0
456
+ ],
457
+ [
458
+ 2.1,
459
+ 279.5,
460
+ 574.5,
461
+ 28.0,
462
+ 0.0
463
+ ],
464
+ [
465
+ 0.0,
466
+ 3.3,
467
+ 34.1,
468
+ 42.3,
469
+ 0.0
470
+ ],
471
+ [
472
+ 0.0,
473
+ 0.0,
474
+ 0.0,
475
+ 0.1,
476
+ 0.0
477
+ ]
478
+ ],
479
+ "support": {
480
+ "trivial": 201,
481
+ "low": 1838,
482
+ "medium": 872,
483
+ "high": 189,
484
+ "expert": 1
485
+ }
486
+ },
487
+ "task": {
488
+ "accuracy": 0.8006841010532753,
489
+ "macro_f1": 0.6323733904733824,
490
+ "macro_f1_present": 0.6323733904733824,
491
+ "per_class": {
492
+ "information_seeking": {
493
+ "f1": 0.8480068752176662,
494
+ "support": 1068
495
+ },
496
+ "reasoning": {
497
+ "f1": 0.5097422442799173,
498
+ "support": 48
499
+ },
500
+ "planning": {
501
+ "f1": 0.5885036465886375,
502
+ "support": 53
503
+ },
504
+ "editing": {
505
+ "f1": 0.8047164777623818,
506
+ "support": 139
507
+ },
508
+ "summarization": {
509
+ "f1": 0.7884833764231882,
510
+ "support": 98
511
+ },
512
+ "extraction": {
513
+ "f1": 0.7743659734172523,
514
+ "support": 180
515
+ },
516
+ "classification": {
517
+ "f1": 0.755260299322683,
518
+ "support": 47
519
+ },
520
+ "translation": {
521
+ "f1": 0.9069849377410021,
522
+ "support": 61
523
+ },
524
+ "coding_debugging": {
525
+ "f1": 0.8598117983995479,
526
+ "support": 246
527
+ },
528
+ "math": {
529
+ "f1": 0.7029567273827195,
530
+ "support": 41
531
+ },
532
+ "data_analysis": {
533
+ "f1": 0.13601164014439981,
534
+ "support": 5
535
+ },
536
+ "writing": {
537
+ "f1": 0.8599838608390206,
538
+ "support": 705
539
+ },
540
+ "role_playing": {
541
+ "f1": 0.699100469772375,
542
+ "support": 30
543
+ },
544
+ "advice_seeking": {
545
+ "f1": 0.586485551349588,
546
+ "support": 75
547
+ },
548
+ "brainstorming": {
549
+ "f1": 0.5322990203969951,
550
+ "support": 83
551
+ },
552
+ "completion": {
553
+ "f1": 0.7009731628974076,
554
+ "support": 162
555
+ },
556
+ "action_execution": {
557
+ "f1": 0.20741165161792083,
558
+ "support": 8
559
+ },
560
+ "chat": {
561
+ "f1": 0.44317185671144815,
562
+ "support": 47
563
+ },
564
+ "other": {
565
+ "f1": 0.3108248487301116,
566
+ "support": 34
567
+ }
568
+ }
569
+ },
570
+ "needs_tools": {
571
+ "f1": 0.4231775842460709,
572
+ "auroc": 0.9139293957758868,
573
+ "positives": 153
574
+ }
575
+ }
576
+ },
577
+ "agreement_with_reference": {
578
+ "max_abs_logit_difference": 19.687698364257812,
579
+ "mean_abs_logit_difference": 2.079862594604492,
580
+ "same_task": 0.8133903133903134,
581
+ "same_needs_tools": 0.967032967032967,
582
+ "same_complexity_tier": 0.7964997964997965,
583
+ "same_policy_tier": 0.8085063085063086
584
+ },
585
+ "agreement_with_parent": {
586
+ "max_abs_logit_difference": 0.0420689582824707,
587
+ "mean_abs_logit_difference": 0.004016549792140722,
588
+ "same_task": 0.9991859991859992,
589
+ "same_needs_tools": 0.9997964997964998,
590
+ "same_complexity_tier": 0.9971509971509972,
591
+ "same_policy_tier": 0.9991859991859992
592
+ },
593
+ "latency_ms_batch1": {
594
+ "p50": 3.0535839978256263,
595
+ "p95": 8.006165997358039,
596
+ "mean": 4.419368929666234,
597
+ "prompts": 200,
598
+ "device": "mps"
599
+ }
600
+ }
footprint.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"peak_rss_mb": 1011.13856, "prompts": 50, "device": "cpu", "backend": "torch"}
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:537baebd93229908312fa5ade0d0a2e98e08a86714b763a7896c3e950bb233d9
3
+ size 48099450
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aebee76d0312011b0d73b08a255d4d11d25c83e9639324870c7b916ea102e13e
3
+ size 34363287
tokenizer_config.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "bos_token": "<bos>",
4
+ "clean_up_tokenization_spaces": false,
5
+ "cls_token": "<bos>",
6
+ "eos_token": "<eos>",
7
+ "extra_special_tokens": [
8
+ "<start_of_turn>",
9
+ "<end_of_turn>"
10
+ ],
11
+ "is_local": true,
12
+ "local_files_only": false,
13
+ "mask_token": "<mask>",
14
+ "max_length": 1024,
15
+ "model_input_names": [
16
+ "input_ids",
17
+ "attention_mask"
18
+ ],
19
+ "model_max_length": 8192,
20
+ "pad_token": "<pad>",
21
+ "padding_side": "right",
22
+ "sep_token": "<eos>",
23
+ "spaces_between_special_tokens": false,
24
+ "stride": 0,
25
+ "tokenizer_class": "TokenizersBackend",
26
+ "truncation_side": "right",
27
+ "truncation_strategy": "longest_first",
28
+ "unk_token": "<unk>"
29
+ }
variant.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "torch",
3
+ "recipe": "torch fp16",
4
+ "source": "trained in this project (pipeline/05_train.py, pipeline/07_variants.py)",
5
+ "parameters": 24047637
6
+ }
vocab_map.npy ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3466512656f2b9ca115751505e310b799aa7a2028bf36d43e6b06413b608e104
3
+ size 2048128