Text Classification
MLX
Safetensors
modernbert
routing
llm-routing
prompt-classification
multilingual
Instructions to use scombat/s1d1-router-6src-compact-mlx-8bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use scombat/s1d1-router-6src-compact-mlx-8bit with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download scombat/s1d1-router-6src-compact-mlx-8bit --local-dir s1d1-router-6src-compact-mlx-8bit
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
Download eval.json from scombat/s1d1-router-6src-compact-mlx-8bit: direct link, hf CLI and curl.
- Browser
- Download file 11.1 kB
-
https://huggingface.co/scombat/s1d1-router-6src-compact-mlx-8bit/resolve/main/eval.json
- Command line
-
hf download hf://scombat/s1d1-router-6src-compact-mlx-8bit/eval.json
-
curl -L -o eval.json https://huggingface.co/scombat/s1d1-router-6src-compact-mlx-8bit/resolve/main/eval.json
11.1 kB
| { | |
| "variant": "v4_vocab32k-mlx-8bit", | |
| "backend": "mlx", | |
| "size_bytes": 181239101, | |
| "validation_all": { | |
| "unweighted": { | |
| "rows": 4914, | |
| "weighted": false, | |
| "complexity": { | |
| "rows": 4850, | |
| "mae": 0.07598711761813803, | |
| "spearman": 0.7325308792002598, | |
| "qwk_rounded": 0.7541114941160277, | |
| "confusion_true_by_pred": [ | |
| [ | |
| 144.0, | |
| 158.0, | |
| 6.0, | |
| 0.0, | |
| 0.0 | |
| ], | |
| [ | |
| 38.0, | |
| 2572.0, | |
| 267.0, | |
| 1.0, | |
| 0.0 | |
| ], | |
| [ | |
| 4.0, | |
| 395.0, | |
| 890.0, | |
| 56.0, | |
| 0.0 | |
| ], | |
| [ | |
| 1.0, | |
| 10.0, | |
| 74.0, | |
| 226.0, | |
| 0.0 | |
| ], | |
| [ | |
| 0.0, | |
| 0.0, | |
| 2.0, | |
| 2.0, | |
| 4.0 | |
| ] | |
| ], | |
| "support": { | |
| "trivial": 308, | |
| "low": 2878, | |
| "medium": 1345, | |
| "high": 311, | |
| "expert": 8 | |
| } | |
| }, | |
| "task": { | |
| "accuracy": 0.8376068376068376, | |
| "macro_f1": 0.7475081932159393, | |
| "macro_f1_present": 0.7475081932159393, | |
| "per_class": { | |
| "information_seeking": { | |
| "f1": 0.8682170542635659, | |
| "support": 1528 | |
| }, | |
| "reasoning": { | |
| "f1": 0.5679012345679012, | |
| "support": 92 | |
| }, | |
| "planning": { | |
| "f1": 0.7132867132867133, | |
| "support": 81 | |
| }, | |
| "editing": { | |
| "f1": 0.8702290076335878, | |
| "support": 193 | |
| }, | |
| "summarization": { | |
| "f1": 0.8571428571428571, | |
| "support": 128 | |
| }, | |
| "extraction": { | |
| "f1": 0.7965738758029979, | |
| "support": 229 | |
| }, | |
| "classification": { | |
| "f1": 0.8901098901098901, | |
| "support": 89 | |
| }, | |
| "translation": { | |
| "f1": 0.9166666666666666, | |
| "support": 73 | |
| }, | |
| "coding_debugging": { | |
| "f1": 0.8972375690607735, | |
| "support": 453 | |
| }, | |
| "math": { | |
| "f1": 0.8704453441295547, | |
| "support": 236 | |
| }, | |
| "data_analysis": { | |
| "f1": 0.72, | |
| "support": 22 | |
| }, | |
| "writing": { | |
| "f1": 0.9018218623481782, | |
| "support": 991 | |
| }, | |
| "role_playing": { | |
| "f1": 0.8771929824561403, | |
| "support": 83 | |
| }, | |
| "advice_seeking": { | |
| "f1": 0.7197452229299363, | |
| "support": 157 | |
| }, | |
| "brainstorming": { | |
| "f1": 0.5863453815261044, | |
| "support": 134 | |
| }, | |
| "completion": { | |
| "f1": 0.7564575645756457, | |
| "support": 267 | |
| }, | |
| "action_execution": { | |
| "f1": 0.5263157894736842, | |
| "support": 11 | |
| }, | |
| "chat": { | |
| "f1": 0.5679012345679012, | |
| "support": 91 | |
| }, | |
| "other": { | |
| "f1": 0.29906542056074764, | |
| "support": 56 | |
| } | |
| } | |
| }, | |
| "needs_tools": { | |
| "f1": 0.6178489702517163, | |
| "auroc": 0.9482045454545455, | |
| "positives": 250 | |
| } | |
| }, | |
| "weighted": { | |
| "rows": 4914, | |
| "weighted": true, | |
| "complexity": { | |
| "rows": 4850, | |
| "mae": 0.07375405856631606, | |
| "spearman": 0.6926722884622424, | |
| "qwk_rounded": 0.7033805871701464, | |
| "confusion_true_by_pred": [ | |
| [ | |
| 138.1, | |
| 155.3, | |
| 5.0, | |
| 0.0, | |
| 0.0 | |
| ], | |
| [ | |
| 40.3, | |
| 2764.5, | |
| 288.3, | |
| 1.1, | |
| 0.0 | |
| ], | |
| [ | |
| 4.1, | |
| 397.4, | |
| 893.8, | |
| 55.9, | |
| 0.0 | |
| ], | |
| [ | |
| 0.4, | |
| 4.1, | |
| 31.4, | |
| 95.4, | |
| 0.0 | |
| ], | |
| [ | |
| 0.0, | |
| 0.0, | |
| 0.3, | |
| 0.3, | |
| 0.6 | |
| ] | |
| ], | |
| "support": { | |
| "trivial": 308, | |
| "low": 2878, | |
| "medium": 1345, | |
| "high": 311, | |
| "expert": 8 | |
| } | |
| }, | |
| "task": { | |
| "accuracy": 0.8394332602849471, | |
| "macro_f1": 0.7340867004427265, | |
| "macro_f1_present": 0.7340867004427265, | |
| "per_class": { | |
| "information_seeking": { | |
| "f1": 0.873880050094105, | |
| "support": 1528 | |
| }, | |
| "reasoning": { | |
| "f1": 0.5754507335128349, | |
| "support": 92 | |
| }, | |
| "planning": { | |
| "f1": 0.6923419444052038, | |
| "support": 81 | |
| }, | |
| "editing": { | |
| "f1": 0.8750917385680073, | |
| "support": 193 | |
| }, | |
| "summarization": { | |
| "f1": 0.8582092786862726, | |
| "support": 128 | |
| }, | |
| "extraction": { | |
| "f1": 0.8006995649406149, | |
| "support": 229 | |
| }, | |
| "classification": { | |
| "f1": 0.8889433148815616, | |
| "support": 89 | |
| }, | |
| "translation": { | |
| "f1": 0.9166481424971284, | |
| "support": 73 | |
| }, | |
| "coding_debugging": { | |
| "f1": 0.898547879675547, | |
| "support": 453 | |
| }, | |
| "math": { | |
| "f1": 0.8655105453546306, | |
| "support": 236 | |
| }, | |
| "data_analysis": { | |
| "f1": 0.6220404272481576, | |
| "support": 22 | |
| }, | |
| "writing": { | |
| "f1": 0.8966525522027537, | |
| "support": 991 | |
| }, | |
| "role_playing": { | |
| "f1": 0.856782491891807, | |
| "support": 83 | |
| }, | |
| "advice_seeking": { | |
| "f1": 0.7325427527778512, | |
| "support": 157 | |
| }, | |
| "brainstorming": { | |
| "f1": 0.5877632803140802, | |
| "support": 134 | |
| }, | |
| "completion": { | |
| "f1": 0.7654770750187783, | |
| "support": 267 | |
| }, | |
| "action_execution": { | |
| "f1": 0.4050722792506393, | |
| "support": 11 | |
| }, | |
| "chat": { | |
| "f1": 0.5710913302828049, | |
| "support": 91 | |
| }, | |
| "other": { | |
| "f1": 0.2649019268090265, | |
| "support": 56 | |
| } | |
| } | |
| }, | |
| "needs_tools": { | |
| "f1": 0.6202797007576061, | |
| "auroc": 0.9485384523441318, | |
| "positives": 250 | |
| } | |
| } | |
| }, | |
| "validation_four_sources": { | |
| "unweighted": { | |
| "rows": 3130, | |
| "weighted": false, | |
| "complexity": { | |
| "rows": 3101, | |
| "mae": 0.07344912939439163, | |
| "spearman": 0.7432828892500157, | |
| "qwk_rounded": 0.7605925596882378, | |
| "confusion_true_by_pred": [ | |
| [ | |
| 98.0, | |
| 99.0, | |
| 4.0, | |
| 0.0, | |
| 0.0 | |
| ], | |
| [ | |
| 25.0, | |
| 1652.0, | |
| 161.0, | |
| 0.0, | |
| 0.0 | |
| ], | |
| [ | |
| 3.0, | |
| 245.0, | |
| 586.0, | |
| 38.0, | |
| 0.0 | |
| ], | |
| [ | |
| 1.0, | |
| 4.0, | |
| 45.0, | |
| 139.0, | |
| 0.0 | |
| ], | |
| [ | |
| 0.0, | |
| 0.0, | |
| 0.0, | |
| 0.0, | |
| 1.0 | |
| ] | |
| ], | |
| "support": { | |
| "trivial": 201, | |
| "low": 1838, | |
| "medium": 872, | |
| "high": 189, | |
| "expert": 1 | |
| } | |
| }, | |
| "task": { | |
| "accuracy": 0.8460063897763578, | |
| "macro_f1": 0.7465511584840749, | |
| "macro_f1_present": 0.7465511584840749, | |
| "per_class": { | |
| "information_seeking": { | |
| "f1": 0.8811835413777162, | |
| "support": 1068 | |
| }, | |
| "reasoning": { | |
| "f1": 0.5747126436781609, | |
| "support": 48 | |
| }, | |
| "planning": { | |
| "f1": 0.651685393258427, | |
| "support": 53 | |
| }, | |
| "editing": { | |
| "f1": 0.8857142857142857, | |
| "support": 139 | |
| }, | |
| "summarization": { | |
| "f1": 0.8631578947368421, | |
| "support": 98 | |
| }, | |
| "extraction": { | |
| "f1": 0.8174386920980926, | |
| "support": 180 | |
| }, | |
| "classification": { | |
| "f1": 0.875, | |
| "support": 47 | |
| }, | |
| "translation": { | |
| "f1": 0.9243697478991597, | |
| "support": 61 | |
| }, | |
| "coding_debugging": { | |
| "f1": 0.9079754601226994, | |
| "support": 246 | |
| }, | |
| "math": { | |
| "f1": 0.7272727272727273, | |
| "support": 41 | |
| }, | |
| "data_analysis": { | |
| "f1": 0.6666666666666666, | |
| "support": 5 | |
| }, | |
| "writing": { | |
| "f1": 0.9019054340155258, | |
| "support": 705 | |
| }, | |
| "role_playing": { | |
| "f1": 0.9152542372881356, | |
| "support": 30 | |
| }, | |
| "advice_seeking": { | |
| "f1": 0.6901408450704225, | |
| "support": 75 | |
| }, | |
| "brainstorming": { | |
| "f1": 0.55, | |
| "support": 83 | |
| }, | |
| "completion": { | |
| "f1": 0.7710843373493976, | |
| "support": 162 | |
| }, | |
| "action_execution": { | |
| "f1": 0.625, | |
| "support": 8 | |
| }, | |
| "chat": { | |
| "f1": 0.5977011494252874, | |
| "support": 47 | |
| }, | |
| "other": { | |
| "f1": 0.3582089552238806, | |
| "support": 34 | |
| } | |
| } | |
| }, | |
| "needs_tools": { | |
| "f1": 0.5808823529411765, | |
| "auroc": 0.9604978034209988, | |
| "positives": 153 | |
| } | |
| }, | |
| "weighted": { | |
| "rows": 3130, | |
| "weighted": true, | |
| "complexity": { | |
| "rows": 3101, | |
| "mae": 0.07160471797212513, | |
| "spearman": 0.704867234170456, | |
| "qwk_rounded": 0.7130554560684663, | |
| "confusion_true_by_pred": [ | |
| [ | |
| 93.8, | |
| 98.4, | |
| 3.5, | |
| 0.0, | |
| 0.0 | |
| ], | |
| [ | |
| 26.1, | |
| 1789.3, | |
| 176.0, | |
| 0.0, | |
| 0.0 | |
| ], | |
| [ | |
| 3.1, | |
| 248.3, | |
| 595.1, | |
| 37.6, | |
| 0.0 | |
| ], | |
| [ | |
| 0.4, | |
| 1.6, | |
| 19.2, | |
| 58.5, | |
| 0.0 | |
| ], | |
| [ | |
| 0.0, | |
| 0.0, | |
| 0.0, | |
| 0.0, | |
| 0.1 | |
| ] | |
| ], | |
| "support": { | |
| "trivial": 201, | |
| "low": 1838, | |
| "medium": 872, | |
| "high": 189, | |
| "expert": 1 | |
| } | |
| }, | |
| "task": { | |
| "accuracy": 0.8481247463011802, | |
| "macro_f1": 0.7286895651247581, | |
| "macro_f1_present": 0.7286895651247581, | |
| "per_class": { | |
| "information_seeking": { | |
| "f1": 0.8851440898353958, | |
| "support": 1068 | |
| }, | |
| "reasoning": { | |
| "f1": 0.5783684198451182, | |
| "support": 48 | |
| }, | |
| "planning": { | |
| "f1": 0.6322924335863169, | |
| "support": 53 | |
| }, | |
| "editing": { | |
| "f1": 0.8913461323150959, | |
| "support": 139 | |
| }, | |
| "summarization": { | |
| "f1": 0.8655264865753218, | |
| "support": 98 | |
| }, | |
| "extraction": { | |
| "f1": 0.8194825792197448, | |
| "support": 180 | |
| }, | |
| "classification": { | |
| "f1": 0.8744675021779321, | |
| "support": 47 | |
| }, | |
| "translation": { | |
| "f1": 0.9243253837334996, | |
| "support": 61 | |
| }, | |
| "coding_debugging": { | |
| "f1": 0.9078078299006189, | |
| "support": 246 | |
| }, | |
| "math": { | |
| "f1": 0.7056980579142964, | |
| "support": 41 | |
| }, | |
| "data_analysis": { | |
| "f1": 0.5512676749948888, | |
| "support": 5 | |
| }, | |
| "writing": { | |
| "f1": 0.8987480780377212, | |
| "support": 705 | |
| }, | |
| "role_playing": { | |
| "f1": 0.9018366831956603, | |
| "support": 30 | |
| }, | |
| "advice_seeking": { | |
| "f1": 0.7056692072538584, | |
| "support": 75 | |
| }, | |
| "brainstorming": { | |
| "f1": 0.5509812080931199, | |
| "support": 83 | |
| }, | |
| "completion": { | |
| "f1": 0.7813712009865946, | |
| "support": 162 | |
| }, | |
| "action_execution": { | |
| "f1": 0.46184553221044683, | |
| "support": 8 | |
| }, | |
| "chat": { | |
| "f1": 0.5921133541119249, | |
| "support": 47 | |
| }, | |
| "other": { | |
| "f1": 0.31680988338284877, | |
| "support": 34 | |
| } | |
| } | |
| }, | |
| "needs_tools": { | |
| "f1": 0.5785238379679463, | |
| "auroc": 0.9630877898213843, | |
| "positives": 153 | |
| } | |
| } | |
| }, | |
| "agreement_with_reference": { | |
| "max_abs_logit_difference": 16.63292694091797, | |
| "mean_abs_logit_difference": 0.7649034857749939, | |
| "same_task": 0.9177859177859178, | |
| "same_needs_tools": 0.9820919820919821, | |
| "same_complexity_tier": 0.9092389092389093, | |
| "same_policy_tier": 0.9151404151404151 | |
| }, | |
| "agreement_with_parent": { | |
| "max_abs_logit_difference": 1.028977870941162, | |
| "mean_abs_logit_difference": 0.06225224584341049, | |
| "same_task": 0.9947089947089947, | |
| "same_needs_tools": 0.9991859991859992, | |
| "same_complexity_tier": 0.99002849002849, | |
| "same_policy_tier": 0.9922669922669922 | |
| }, | |
| "latency_ms_batch1": { | |
| "p50": 5.329958999936935, | |
| "p95": 34.000291998381726, | |
| "mean": 17.72976168012974, | |
| "prompts": 200, | |
| "device": "cpu" | |
| } | |
| } |