System01 v1.0
Browse files- NOTICE +1 -1
- README.md +78 -81
- assets/azerbaijani_categories.png +2 -2
- assets/calibration.png +2 -2
- assets/doom.gif +2 -2
- assets/doom_results.png +0 -0
- assets/multilingual.png +0 -0
- assets/training.png +2 -2
- assets/vs_jev.png +2 -2
- bench/az_categories.py +5 -5
- bench/plots_v2.py +36 -28
- bench/run_bench.py +4 -4
- bench/speed.py +5 -5
- config.json +16 -0
- doom/bench_doom.py +9 -7
- doom/doom_live.py +6 -6
- doom/index.html +9 -6
- model.safetensors +1 -1
- pyproject.toml +4 -4
- results/az_categories.json +35 -35
- results/doom.json +32 -32
- results/final_release.json +390 -78
- results/reliability.json +27 -27
- results/train_curve_v5.jsonl +15 -0
- rl_agent_config.json +11 -13
- {system1 → system01}/__init__.py +1 -1
- {system1 → system01}/agent.py +17 -17
- {system1 → system01}/common.py +1 -1
- {system1 → system01}/email.py +1 -1
- {system1 → system01}/lang.py +1 -1
- {system1 → system01}/presets.py +0 -0
- {system1 → system01}/server.py +11 -11
- {system1 → system01}/shortlist.py +1 -1
- train/soup.py +8 -6
- train/train_v2.py +3 -3
- train/train_v4.py +14 -8
NOTICE
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
|
| 2 |
Copyright 2026 HAL-X
|
| 3 |
|
| 4 |
This product includes software and model weights derived from works licensed under the
|
|
|
|
| 1 |
+
System01
|
| 2 |
Copyright 2026 HAL-X
|
| 3 |
|
| 4 |
This product includes software and model weights derived from works licensed under the
|
README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
---
|
| 2 |
license: apache-2.0
|
| 3 |
-
library_name:
|
| 4 |
pipeline_tag: text-classification
|
| 5 |
language:
|
| 6 |
- az
|
|
@@ -23,7 +23,7 @@ language:
|
|
| 23 |
- it
|
| 24 |
- multilingual
|
| 25 |
tags:
|
| 26 |
-
-
|
| 27 |
- rlcd
|
| 28 |
- calibrated-decisions
|
| 29 |
- decision-model
|
|
@@ -37,25 +37,23 @@ base_model:
|
|
| 37 |
- jhu-clsp/mmBERT-base
|
| 38 |
---
|
| 39 |
|
| 40 |
-
#
|
| 41 |
|
| 42 |
-
**Calibrated decisions in one forward pass: ~9 ms on one RTX 4090, 100+ languages
|
| 43 |
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
System1 is a non-autoregressive decision model. You give it a **state** (a message, ticket, email, JSON record, agent trace or game state) and **typed questions** written in plain language. It returns a typed answer to every question, each with a **calibrated probability**, in a single encoder pass. It generates no text, so there is nothing to parse and nothing to hallucinate. The label set is part of the input, so a new schema needs no retraining.
|
| 47 |
|
| 48 |
<p align="center">
|
| 49 |
-
<img src="assets/doom.gif" width="100%" alt="
|
| 50 |
-
<br/><sub><b>
|
| 51 |
</p>
|
| 52 |
|
| 53 |
| | |
|
| 54 |
|---|---|
|
| 55 |
| **Speed** | **9.4 ms** for one question · **10.1 ms** for a 3-question request · **150 requests in 156 ms** batched · up to **3,075 decisions/s** on one RTX 4090 |
|
| 56 |
-
| **Azerbaijani** | **0.
|
| 57 |
-
| **Multilingual** | 0.
|
| 58 |
-
| **vs TypeSafe Jev** | Higher on AG News (**0.931** vs 0.910) and SMS spam (**0.936** vs 0.930), better calibrated (**ECE 0.02–0.
|
| 59 |
| **Footprint** | 322M parameters (307M mmBERT-base encoder + 15M decision head), **0.65 GB** of VRAM in bf16 |
|
| 60 |
| **License** | Apache 2.0, self-hosted |
|
| 61 |
|
|
@@ -70,14 +68,14 @@ System1 is a non-autoregressive decision model. You give it a **state** (a messa
|
|
| 70 |
## Quickstart
|
| 71 |
|
| 72 |
```bash
|
| 73 |
-
hf download doofz/
|
| 74 |
-
pip install "./
|
| 75 |
```
|
| 76 |
|
| 77 |
```python
|
| 78 |
-
import
|
| 79 |
|
| 80 |
-
model =
|
| 81 |
|
| 82 |
state = "511 maddə nə tənzimləyir" # "What does article 511 regulate?"
|
| 83 |
|
|
@@ -98,8 +96,8 @@ questions = {
|
|
| 98 |
}
|
| 99 |
|
| 100 |
out = model.predict(state, questions)
|
| 101 |
-
out["answers"]["model"] # {'choice': 'legolas-flash', 'probabilities': {'legolas-flash': 0.
|
| 102 |
-
out["answers"]["is_legal"] # {'noul': 0.
|
| 103 |
```
|
| 104 |
|
| 105 |
**Three answer types**
|
|
@@ -117,7 +115,7 @@ Questions can be written in English or Azerbaijani, and the state can be in any
|
|
| 117 |
## REST API
|
| 118 |
|
| 119 |
```bash
|
| 120 |
-
|
| 121 |
```
|
| 122 |
|
| 123 |
| Method | Endpoint | Body → Result |
|
|
@@ -129,7 +127,7 @@ SYSTEM1_API_KEY=change-me CUDA_VISIBLE_DEVICES=0 python -m system1.server --port
|
|
| 129 |
|
| 130 |
```bash
|
| 131 |
curl -s http://localhost:8095/v1/decide \
|
| 132 |
-
-H "Authorization: Bearer $
|
| 133 |
-d '{"state": "cox pis proqramdir, islemir",
|
| 134 |
"questions": {"sentiment": {"type": "choice", "instructions": "Sentiment of this review?",
|
| 135 |
"criteria": ["positive", "negative"]}}}'
|
|
@@ -139,7 +137,7 @@ curl -s http://localhost:8095/v1/decide \
|
|
| 139 |
|
| 140 |
## What it is for
|
| 141 |
|
| 142 |
-
|
| 143 |
|
| 144 |
- **LLM gateway and model routing.** Pick the right model, agent or tool for every request in ~9 ms: a legal model for law questions, a cheap fast model for trivia, a frontier model for hard reasoning. Descriptions are free text, so adding a model is a config change.
|
| 145 |
- **Guardrails before and after the LLM.** Jailbreak, prompt injection, PII, toxicity, self-harm, scam or phishing, and off-topic checks on every prompt and response, in any language, as many checks as you like in one pass.
|
|
@@ -155,70 +153,70 @@ System1 is the fast, cheap, calibrated **System 1** next to your LLMs. It makes
|
|
| 155 |
|
| 156 |
All numbers were measured by HAL-X on one RTX 4090 (bf16) unless marked *published*. **Evaluation data is held out:** the public datasets below were never used in training, and the synthetic test sets use schemas and label sets that do not occur in training. Questions are written in English and the text is in the target language.
|
| 157 |
|
| 158 |
-
###
|
| 159 |
|
| 160 |
-
<p align="center"><img src="assets/vs_jev.png" width="100%" alt="
|
| 161 |
|
| 162 |
-
| dataset | labels |
|
| 163 |
|---|---|---|---|
|
| 164 |
| AG News | 4 | **0.931** | 0.910 |
|
| 165 |
| SMS spam | 2 | **0.936** | 0.930 |
|
| 166 |
-
| DAIR Emotion | 6 | 0.
|
| 167 |
-
| Banking77 | 77 | 0.
|
| 168 |
-
| calibration error (ECE) | | **0.02–0.
|
| 169 |
| p50 latency, 1 question | | **9.4 ms** (local 4090) | 236–276 ms (API) |
|
| 170 |
-
| Azerbaijani | | **0.
|
| 171 |
| weights | | **open (Apache 2.0)** | closed API |
|
| 172 |
|
| 173 |
-
Jev figures are from independent third-party evaluations ([AbdelStark/jev-benchmarks](https://github.com/AbdelStark/jev-benchmarks), class-balanced n=100 per dataset; [nibzard/decision-model-benchmark](https://github.com/nibzard/decision-model-benchmark)).
|
| 174 |
|
| 175 |
### Azerbaijani, by category
|
| 176 |
|
| 177 |
<p align="center"><img src="assets/azerbaijani_categories.png" width="100%" alt="Azerbaijani benchmark by category"/></p>
|
| 178 |
|
| 179 |
-
| category (Azerbaijani) | n | base checkpoint | **
|
| 180 |
|---|---|---|---|---|
|
| 181 |
-
| Guardrails & policy checks (14 checks) | 181 | 0.657 | **0.956** | 0.
|
| 182 |
-
| Model / agent routing | 1042 | 0.327 | **0.
|
| 183 |
-
|
|
| 184 |
-
| Ticket triage · category | 235 | 0.545 | **0.
|
| 185 |
-
|
|
| 186 |
-
| Ticket triage · yes/no flags | 215 | 0.642 | **0.
|
| 187 |
-
|
|
| 188 |
-
|
|
| 189 |
-
| App-review sentiment | 1000 | 0.735 | **0.
|
| 190 |
-
| Voice-assistant intent (MASSIVE, 60 classes) | 1000 | 0.226 | **0.
|
| 191 |
-
|
|
| 192 |
-
|
|
| 193 |
|
| 194 |
### Multilingual
|
| 195 |
|
| 196 |
<p align="center"><img src="assets/multilingual.png" width="100%" alt="MASSIVE in 14 languages"/></p>
|
| 197 |
|
| 198 |
-
| language | MASSIVE domain (18) · base | MASSIVE · **
|
| 199 |
|---|---|---|---|---|
|
| 200 |
-
| az | 0.416 | **0.
|
| 201 |
-
| en | 0.694 | **0.910** | 0.784 | **0.
|
| 202 |
-
| ru | 0.576 | **0.
|
| 203 |
-
| tr | 0.502 | **0.
|
| 204 |
-
| de | 0.564 | **0.
|
| 205 |
-
| fr | 0.598 | **0.
|
| 206 |
| es | 0.550 | **0.878** | – | – |
|
| 207 |
-
| ar | 0.438 | **0.
|
| 208 |
| fa | 0.466 | **0.876** | – | – |
|
| 209 |
-
| hi | 0.462 | **0.
|
| 210 |
-
| zh-CN | 0.610 | **0.
|
| 211 |
-
| ja | 0.592 | **0.
|
| 212 |
-
| ko | 0.488 | **0.
|
| 213 |
-
| ka | 0.168 | **0.
|
| 214 |
-
| kk | – | – | 0.686 | **0.
|
| 215 |
-
| uz | – | – | 0.583 | **0.
|
| 216 |
|
| 217 |
### Calibration
|
| 218 |
|
| 219 |
<p align="center"><img src="assets/calibration.png" width="100%" alt="Calibration"/></p>
|
| 220 |
|
| 221 |
-
Over 5,802 held-out Azerbaijani decisions,
|
| 222 |
|
| 223 |
---
|
| 224 |
|
|
@@ -240,13 +238,13 @@ Measured on one **NVIDIA GeForce RTX 4090** with bf16 weights. Every request car
|
|
| 240 |
| HTTP `/v1/decide`, 1 request | 17.8 ms | – | – |
|
| 241 |
| HTTP `/v1/decide/batch`, 150 items | 170 ms | 1.13 ms | 883 req/s |
|
| 242 |
|
| 243 |
-
Through the HTTP API, one **`/v1/decide/batch` with 150 items takes 170 ms**. 150 *concurrent* single `/v1/decide` calls are micro-batched on the server (3 ms window) and finish in about 1.3 s, dominated by HTTP and JSON handling rather than the model. Use the batch endpoint for bulk traffic.
|
| 244 |
|
| 245 |
---
|
| 246 |
|
| 247 |
## Doom demo
|
| 248 |
|
| 249 |
-
|
| 250 |
|
| 251 |
```
|
| 252 |
crosshair: empty; nearest VISIBLE enemy is 34deg to the LEFT
|
|
@@ -259,32 +257,32 @@ It answers `choice` (which key to press; each option's description states when i
|
|
| 259 |
|
| 260 |
<p align="center"><img src="assets/doom_results.png" width="100%" alt="Doom results"/></p>
|
| 261 |
|
| 262 |
-
| scenario |
|
| 263 |
|---|---|---|---|---|
|
| 264 |
-
| `defend_the_center` | **13.
|
| 265 |
-
| `defend_the_line` | **
|
| 266 |
-
| `deadly_corridor` | **
|
| 267 |
| `basic` | **1.0** ± 0.0 | 0.8 ± 0.4 | **79** | -835 |
|
| 268 |
|
| 269 |
-
20 episodes per scenario at skill 2, with a decision every 4 tics; p50 latency is 10 ms per decision. The key pressed is `ATTACK` when
|
| 270 |
|
| 271 |
Run it yourself with a live browser dashboard: the game on the left, the model's decision probabilities, the crosshair probability, latency and episode statistics on the right.
|
| 272 |
|
| 273 |
```bash
|
| 274 |
pip install vizdoom
|
| 275 |
-
python doom/doom_live.py --model doofz/
|
| 276 |
-
python doom/bench_doom.py --model doofz/
|
| 277 |
```
|
| 278 |
|
| 279 |
---
|
| 280 |
|
| 281 |
## Post-training
|
| 282 |
|
| 283 |
-
|
| 284 |
|
| 285 |
<p align="center"><img src="assets/training.png" width="100%" alt="Training curves"/></p>
|
| 286 |
|
| 287 |
-
**Stage 1: multilingual and Azerbaijani decisions (158k decisions, 2 epochs
|
| 288 |
|
| 289 |
| source | decisions | notes |
|
| 290 |
|---|---|---|
|
|
@@ -293,12 +291,14 @@ System1 starts from a multilingual decision checkpoint on an **mmBERT-base** enc
|
|
| 293 |
| synthetic fine-grained taxonomies | 27k | 25–60 near-neighbour intents per schema |
|
| 294 |
| synthetic emotion and tone | 11k | 3–10 labels per schema |
|
| 295 |
| synthetic guardrails | 8k | 14 policy checks with hard negatives |
|
| 296 |
-
|
|
| 297 |
-
| Azerbaijani app reviews | 5k | sentiment (choice + yes/no) |
|
|
|
|
|
|
|
| 298 |
|
| 299 |
-
|
| 300 |
|
| 301 |
-
**Stage
|
| 302 |
|
| 303 |
**Objective (RLCD).** The model reports a distribution $q$ over the options and maximises a strictly proper scoring rule, so honest probabilities are the unique optimum:
|
| 304 |
|
|
@@ -336,11 +336,8 @@ The head's token budget grows with the number of options (up to half of the 1,02
|
|
| 336 |
|
| 337 |
## Honest limits
|
| 338 |
|
| 339 |
-
- **Very large label sets.** On 77-way Banking77,
|
| 340 |
-
- **Noisy emotion labels.** On DAIR Emotion, whose labels come from hashtags, accuracy is about 0.45 and calibration is weaker (ECE 0.
|
| 341 |
-
- **Ordinal `score` questions** are the weakest type (0.76 on Azerbaijani severity scales).
|
| 342 |
-
- **Synthetic evaluation.** Several category scores use held-out synthetic schemas written by an LLM teacher and verified by an LLM judge. The public datasets (MASSIVE, SIB-200, AG News, Banking77, SMS spam, DAIR Emotion, app reviews) are the independent check.
|
| 343 |
-
- **Doom.** It plays zero-shot, misses shots and dies on harder maps. The demo is about speed and calibrated control, not superhuman play.
|
| 344 |
|
| 345 |
---
|
| 346 |
|
|
@@ -349,10 +346,10 @@ The head's token budget grows with the number of options (up to half of the 1,02
|
|
| 349 |
Apache 2.0. See `LICENSE` and `NOTICE`.
|
| 350 |
|
| 351 |
```bibtex
|
| 352 |
-
@misc{
|
| 353 |
-
title = {
|
| 354 |
author = {HAL-X},
|
| 355 |
year = {2026},
|
| 356 |
-
url = {https://huggingface.co/doofz/
|
| 357 |
}
|
| 358 |
```
|
|
|
|
| 1 |
---
|
| 2 |
license: apache-2.0
|
| 3 |
+
library_name: system01
|
| 4 |
pipeline_tag: text-classification
|
| 5 |
language:
|
| 6 |
- az
|
|
|
|
| 23 |
- it
|
| 24 |
- multilingual
|
| 25 |
tags:
|
| 26 |
+
- system01
|
| 27 |
- rlcd
|
| 28 |
- calibrated-decisions
|
| 29 |
- decision-model
|
|
|
|
| 37 |
- jhu-clsp/mmBERT-base
|
| 38 |
---
|
| 39 |
|
| 40 |
+
# System01
|
| 41 |
|
| 42 |
+
**Calibrated decisions in one forward pass: ~9 ms on one RTX 4090, 100+ languages.**
|
| 43 |
|
| 44 |
+
System01 is a non-autoregressive decision model. You give it a **state** (a message, ticket, email, JSON record, agent trace or game state) and **typed questions** written in plain language. It returns a typed answer to every question, each with a **calibrated probability**, in a single encoder pass. It generates no text, so there is nothing to parse and nothing to hallucinate. The label set is part of the input, so a new schema needs no retraining.
|
|
|
|
|
|
|
| 45 |
|
| 46 |
<p align="center">
|
| 47 |
+
<img src="assets/doom.gif" width="100%" alt="System01 playing Doom from a text description of the game state"/>
|
| 48 |
+
<br/><sub><b>System01 plays Doom with no vision and no text generation.</b> Every 4 game tics the engine state is written as text. System01 answers two typed questions in one pass: a <code>choice</code> over the actions and a <code>noul</code> "is the crosshair on an enemy?". About 10 ms per decision, and no Doom data was used in training.</sub>
|
| 49 |
</p>
|
| 50 |
|
| 51 |
| | |
|
| 52 |
|---|---|
|
| 53 |
| **Speed** | **9.4 ms** for one question · **10.1 ms** for a 3-question request · **150 requests in 156 ms** batched · up to **3,075 decisions/s** on one RTX 4090 |
|
| 54 |
+
| **Azerbaijani** | **0.875** accuracy over 12 task categories (base checkpoint: 0.440). ECE **0.015** (base: 0.268). |
|
| 55 |
+
| **Multilingual** | 0.77–0.91 on 18-way intent-domain classification in 14 languages. Post-trained on 18 languages. |
|
| 56 |
+
| **vs TypeSafe Jev** | Higher on AG News (**0.931** vs 0.910) and SMS spam (**0.936** vs 0.930), better calibrated (**ECE 0.02–0.03** vs 0.246), **~25× lower latency**, open weights |
|
| 57 |
| **Footprint** | 322M parameters (307M mmBERT-base encoder + 15M decision head), **0.65 GB** of VRAM in bf16 |
|
| 58 |
| **License** | Apache 2.0, self-hosted |
|
| 59 |
|
|
|
|
| 68 |
## Quickstart
|
| 69 |
|
| 70 |
```bash
|
| 71 |
+
hf download doofz/System01 --include "system01/*" "pyproject.toml" "setup.py" "README.md" --local-dir System01-src
|
| 72 |
+
pip install "./System01-src[server]"
|
| 73 |
```
|
| 74 |
|
| 75 |
```python
|
| 76 |
+
import system01
|
| 77 |
|
| 78 |
+
model = system01.load("doofz/System01", device="cuda")
|
| 79 |
|
| 80 |
state = "511 maddə nə tənzimləyir" # "What does article 511 regulate?"
|
| 81 |
|
|
|
|
| 96 |
}
|
| 97 |
|
| 98 |
out = model.predict(state, questions)
|
| 99 |
+
out["answers"]["model"] # {'choice': 'legolas-flash', 'probabilities': {'legolas-flash': 0.62, ...}, 'confidence': ...}
|
| 100 |
+
out["answers"]["is_legal"] # {'noul': 0.99, ...}
|
| 101 |
```
|
| 102 |
|
| 103 |
**Three answer types**
|
|
|
|
| 115 |
## REST API
|
| 116 |
|
| 117 |
```bash
|
| 118 |
+
SYSTEM01_API_KEY=change-me CUDA_VISIBLE_DEVICES=0 python -m system01.server --port 8095
|
| 119 |
```
|
| 120 |
|
| 121 |
| Method | Endpoint | Body → Result |
|
|
|
|
| 127 |
|
| 128 |
```bash
|
| 129 |
curl -s http://localhost:8095/v1/decide \
|
| 130 |
+
-H "Authorization: Bearer $SYSTEM01_API_KEY" -H "Content-Type: application/json" \
|
| 131 |
-d '{"state": "cox pis proqramdir, islemir",
|
| 132 |
"questions": {"sentiment": {"type": "choice", "instructions": "Sentiment of this review?",
|
| 133 |
"criteria": ["positive", "negative"]}}}'
|
|
|
|
| 137 |
|
| 138 |
## What it is for
|
| 139 |
|
| 140 |
+
System01 is the fast, cheap, calibrated **System 1** next to your LLMs. It makes the high-volume decisions and hands a request to a large model only when its own confidence says it should.
|
| 141 |
|
| 142 |
- **LLM gateway and model routing.** Pick the right model, agent or tool for every request in ~9 ms: a legal model for law questions, a cheap fast model for trivia, a frontier model for hard reasoning. Descriptions are free text, so adding a model is a config change.
|
| 143 |
- **Guardrails before and after the LLM.** Jailbreak, prompt injection, PII, toxicity, self-harm, scam or phishing, and off-topic checks on every prompt and response, in any language, as many checks as you like in one pass.
|
|
|
|
| 153 |
|
| 154 |
All numbers were measured by HAL-X on one RTX 4090 (bf16) unless marked *published*. **Evaluation data is held out:** the public datasets below were never used in training, and the synthetic test sets use schemas and label sets that do not occur in training. Questions are written in English and the text is in the target language.
|
| 155 |
|
| 156 |
+
### System01 vs TypeSafe Jev
|
| 157 |
|
| 158 |
+
<p align="center"><img src="assets/vs_jev.png" width="100%" alt="System01 vs Jev"/></p>
|
| 159 |
|
| 160 |
+
| dataset | labels | System01 | Jev 1.13.0 *(published)* |
|
| 161 |
|---|---|---|---|
|
| 162 |
| AG News | 4 | **0.931** | 0.910 |
|
| 163 |
| SMS spam | 2 | **0.936** | 0.930 |
|
| 164 |
+
| DAIR Emotion | 6 | 0.449 | **0.480** |
|
| 165 |
+
| Banking77 | 77 | 0.622 | **0.763** (77 labels) / 0.870 (72 labels) |
|
| 166 |
+
| calibration error (ECE) | | **0.02–0.03** | 0.246 |
|
| 167 |
| p50 latency, 1 question | | **9.4 ms** (local 4090) | 236–276 ms (API) |
|
| 168 |
+
| Azerbaijani | | **0.875** (12 categories) | not published |
|
| 169 |
| weights | | **open (Apache 2.0)** | closed API |
|
| 170 |
|
| 171 |
+
Jev figures are from independent third-party evaluations ([AbdelStark/jev-benchmarks](https://github.com/AbdelStark/jev-benchmarks), class-balanced n=100 per dataset; [nibzard/decision-model-benchmark](https://github.com/nibzard/decision-model-benchmark)). System01 uses about 1,000 class-balanced test examples per dataset. Both are zero-shot, and both are given label descriptions. System01's wording for Emotion and SMS was chosen on the *validation* splits before the single test run. Jev still leads on DAIR Emotion (inside its ±0.10 confidence interval at n=100) and clearly on 77-way Banking77, where System01 uses its two-pass tournament. Jev's latency includes a network round trip.
|
| 172 |
|
| 173 |
### Azerbaijani, by category
|
| 174 |
|
| 175 |
<p align="center"><img src="assets/azerbaijani_categories.png" width="100%" alt="Azerbaijani benchmark by category"/></p>
|
| 176 |
|
| 177 |
+
| category (Azerbaijani) | n | base checkpoint | **System01** | ECE |
|
| 178 |
|---|---|---|---|---|
|
| 179 |
+
| Guardrails & policy checks (14 checks) | 181 | 0.657 | **0.956** | 0.036 |
|
| 180 |
+
| Model / agent routing | 1042 | 0.327 | **0.951** | 0.013 |
|
| 181 |
+
| LLM routing, hand-written | 32 | 0.250 | **0.938** | 0.077 |
|
| 182 |
+
| Ticket triage · category | 235 | 0.545 | **0.932** | 0.034 |
|
| 183 |
+
| Emotion & tone | 269 | 0.502 | **0.922** | 0.044 |
|
| 184 |
+
| Ticket triage · yes/no flags | 215 | 0.642 | **0.902** | 0.049 |
|
| 185 |
+
| Fine-grained intent (25-60 classes) | 431 | 0.237 | **0.889** | 0.042 |
|
| 186 |
+
| Voice-assistant domain (MASSIVE, 18 classes) | 1000 | 0.404 | **0.885** | 0.044 |
|
| 187 |
+
| App-review sentiment | 1000 | 0.735 | **0.849** | 0.091 |
|
| 188 |
+
| Voice-assistant intent (MASSIVE, 60 classes) | 1000 | 0.226 | **0.818** | 0.031 |
|
| 189 |
+
| News topic (SIB-200, zero-shot) | 204 | 0.672 | **0.740** | 0.138 |
|
| 190 |
+
| Ticket triage · severity scale | 193 | 0.321 | **0.736** | 0.065 |
|
| 191 |
|
| 192 |
### Multilingual
|
| 193 |
|
| 194 |
<p align="center"><img src="assets/multilingual.png" width="100%" alt="MASSIVE in 14 languages"/></p>
|
| 195 |
|
| 196 |
+
| language | MASSIVE domain (18) · base | MASSIVE · **System01** | SIB-200 topic (7, zero-shot) · base | SIB-200 · **System01** |
|
| 197 |
|---|---|---|---|---|
|
| 198 |
+
| az | 0.416 | **0.882** | 0.672 | **0.740** |
|
| 199 |
+
| en | 0.694 | **0.910** | 0.784 | **0.848** |
|
| 200 |
+
| ru | 0.576 | **0.906** | 0.755 | **0.784** |
|
| 201 |
+
| tr | 0.502 | **0.882** | 0.740 | **0.770** |
|
| 202 |
+
| de | 0.564 | **0.868** | 0.755 | **0.819** |
|
| 203 |
+
| fr | 0.598 | **0.906** | – | – |
|
| 204 |
| es | 0.550 | **0.878** | – | – |
|
| 205 |
+
| ar | 0.438 | **0.818** | 0.730 | **0.779** |
|
| 206 |
| fa | 0.466 | **0.876** | – | – |
|
| 207 |
+
| hi | 0.462 | **0.846** | 0.672 | **0.716** |
|
| 208 |
+
| zh-CN | 0.610 | **0.870** | 0.779 | **0.794** |
|
| 209 |
+
| ja | 0.592 | **0.906** | – | – |
|
| 210 |
+
| ko | 0.488 | **0.872** | – | – |
|
| 211 |
+
| ka | 0.168 | **0.774** | – | – |
|
| 212 |
+
| kk | – | – | 0.686 | **0.750** |
|
| 213 |
+
| uz | – | – | 0.583 | **0.657** |
|
| 214 |
|
| 215 |
### Calibration
|
| 216 |
|
| 217 |
<p align="center"><img src="assets/calibration.png" width="100%" alt="Calibration"/></p>
|
| 218 |
|
| 219 |
+
Over 5,802 held-out Azerbaijani decisions, System01's reported confidence tracks observed accuracy: **ECE 0.015**, against 0.268 before post-training and 0.246 published for Jev.
|
| 220 |
|
| 221 |
---
|
| 222 |
|
|
|
|
| 238 |
| HTTP `/v1/decide`, 1 request | 17.8 ms | – | – |
|
| 239 |
| HTTP `/v1/decide/batch`, 150 items | 170 ms | 1.13 ms | 883 req/s |
|
| 240 |
|
| 241 |
+
Through the HTTP API, one **`/v1/decide/batch` with 150 items takes 170 ms**. 150 *concurrent* single `/v1/decide` calls are micro-batched on the server (3 ms window) and finish in about 1.3 s, dominated by HTTP and JSON handling rather than the model. Use the batch endpoint for bulk traffic. System01 emits zero output tokens, so latency does not grow with answer length. An LLM producing the same JSON decision would spend hundreds of milliseconds just decoding it.
|
| 242 |
|
| 243 |
---
|
| 244 |
|
| 245 |
## Doom demo
|
| 246 |
|
| 247 |
+
System01 plays [ViZDoom](https://github.com/Farama-Foundation/ViZDoom) **from text only**. Each step, the game state is serialised:
|
| 248 |
|
| 249 |
```
|
| 250 |
crosshair: empty; nearest VISIBLE enemy is 34deg to the LEFT
|
|
|
|
| 257 |
|
| 258 |
<p align="center"><img src="assets/doom_results.png" width="100%" alt="Doom results"/></p>
|
| 259 |
|
| 260 |
+
| scenario | System01 kills / episode | random policy | System01 reward | random reward |
|
| 261 |
|---|---|---|---|---|
|
| 262 |
+
| `defend_the_center` | **13.6** ± 1.9 | 1.4 ± 1.4 | **13** | 0 |
|
| 263 |
+
| `defend_the_line` | **10.7** ± 2.7 | 7.7 ± 3.4 | **10** | 7 |
|
| 264 |
+
| `deadly_corridor` | **4.5** ± 1.1 | 0.8 ± 0.6 | **699** | -72 |
|
| 265 |
| `basic` | **1.0** ± 0.0 | 0.8 ± 0.4 | **79** | -835 |
|
| 266 |
|
| 267 |
+
20 episodes per scenario at skill 2, with a decision every 4 tics; p50 latency is 10 ms per decision. The key pressed is `ATTACK` when System01's `noul` probability that the crosshair is on an enemy is ≥ 0.5, and otherwise its `choice`. In `basic`, System01 kills the target in about 6 steps (reward +79 vs −835 for random). In `deadly_corridor` it survives 13× longer than random but does not reach the armor.
|
| 268 |
|
| 269 |
Run it yourself with a live browser dashboard: the game on the left, the model's decision probabilities, the crosshair probability, latency and episode statistics on the right.
|
| 270 |
|
| 271 |
```bash
|
| 272 |
pip install vizdoom
|
| 273 |
+
python doom/doom_live.py --model doofz/System01 --port 8096 # open http://localhost:8096
|
| 274 |
+
python doom/bench_doom.py --model doofz/System01 --episodes 10 # headless benchmark vs a random policy
|
| 275 |
```
|
| 276 |
|
| 277 |
---
|
| 278 |
|
| 279 |
## Post-training
|
| 280 |
|
| 281 |
+
System01 starts from a multilingual decision checkpoint on an **mmBERT-base** encoder. HAL-X then post-trained it in two stages on one RTX 4090.
|
| 282 |
|
| 283 |
<p align="center"><img src="assets/training.png" width="100%" alt="Training curves"/></p>
|
| 284 |
|
| 285 |
+
**Stage 1: multilingual and Azerbaijani decisions (158k decisions, 2 epochs).**
|
| 286 |
|
| 287 |
| source | decisions | notes |
|
| 288 |
|---|---|---|
|
|
|
|
| 291 |
| synthetic fine-grained taxonomies | 27k | 25–60 near-neighbour intents per schema |
|
| 292 |
| synthetic emotion and tone | 11k | 3–10 labels per schema |
|
| 293 |
| synthetic guardrails | 8k | 14 policy checks with hard negatives |
|
| 294 |
+
| real curated data (data team) · MASSIVE, 14 languages | 45k | domain, intent (5–60 options) and intent yes/no |
|
| 295 |
+
| real curated data (data team) · Azerbaijani app reviews | 5k | sentiment (choice + yes/no) |
|
| 296 |
+
|
| 297 |
+
Training combines **real data curated by our data team** (public corpora cleaned, relabelled into typed questions and class-balanced) with synthetic data. The synthetic data covers **16 languages, 38% Azerbaijani** (including text typed without Azerbaijani letters). It is produced **label-first**: the gold answer is sampled, and then a teacher LLM writes a message that makes it true. An **independent judge pass** answers the same questions blind, and any answer the judge disagrees with is discarded (about 20% overall). In total 136k synthetic decisions survived.
|
| 298 |
|
| 299 |
+
**Stage 2: spatial control (8k new + 45k replayed decisions, 1 epoch).** Procedurally generated tasks (camera, UI cursor, drone, robot arm, car, telescope) teach "the target is left, right, centred or not visible → which control". Replay of stage-1 data keeps other skills intact (mean change across 39 benchmarks: −0.001). No game data is involved; this is the skill the Doom demo transfers.
|
| 300 |
|
| 301 |
+
**Stage 3: routing emphasis (48k decisions, 1 epoch).** A short pass over LLM/agent-routing schemas (many with very short queries) plus spatial and general replay. It recovers routing accuracy that stage 2 had diluted and keeps the Doom skill; mean change across 39 benchmarks: −0.001.
|
| 302 |
|
| 303 |
**Objective (RLCD).** The model reports a distribution $q$ over the options and maximises a strictly proper scoring rule, so honest probabilities are the unique optimum:
|
| 304 |
|
|
|
|
| 336 |
|
| 337 |
## Honest limits
|
| 338 |
|
| 339 |
+
- **Very large label sets.** On 77-way Banking77, System01 reaches 0.622, below Jev. For 50+ labels, supply short, distinctive descriptions, or split the question into a coarse-to-fine hierarchy.
|
| 340 |
+
- **Noisy emotion labels.** On DAIR Emotion, whose labels come from hashtags, accuracy is about 0.45 and calibration is weaker (ECE 0.34). On clean emotion data it reaches 0.94.
|
|
|
|
|
|
|
|
|
|
| 341 |
|
| 342 |
---
|
| 343 |
|
|
|
|
| 346 |
Apache 2.0. See `LICENSE` and `NOTICE`.
|
| 347 |
|
| 348 |
```bibtex
|
| 349 |
+
@misc{halx2026system01,
|
| 350 |
+
title = {System01: Calibrated Non-Autoregressive Decisions in 100+ Languages},
|
| 351 |
author = {HAL-X},
|
| 352 |
year = {2026},
|
| 353 |
+
url = {https://huggingface.co/doofz/System01}
|
| 354 |
}
|
| 355 |
```
|
assets/azerbaijani_categories.png
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
assets/calibration.png
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
assets/doom.gif
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
assets/doom_results.png
CHANGED
|
|
assets/multilingual.png
CHANGED
|
|
assets/training.png
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
assets/vs_jev.png
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
bench/az_categories.py
CHANGED
|
@@ -3,8 +3,8 @@ import json, sys
|
|
| 3 |
from collections import defaultdict
|
| 4 |
import numpy as np
|
| 5 |
sys.path.insert(0, "."); sys.path.insert(0, "./bench")
|
| 6 |
-
import
|
| 7 |
-
from
|
| 8 |
|
| 9 |
SYN = "./data/synth"
|
| 10 |
GUARD_NAMES = {"jailbreak": "Jailbreak detection", "prompt_injection": "Prompt-injection detection", "pii": "PII detection",
|
|
@@ -37,7 +37,7 @@ def cats():
|
|
| 37 |
c["Voice-assistant domain (MASSIVE, 18 classes)"] = tasks.massive_scenario("az", "test", 1000)
|
| 38 |
c["App-review sentiment"] = tasks.az_sentiment("test", 1000)
|
| 39 |
c["News topic (SIB-200, zero-shot)"] = tasks.sib_any("az", "test")
|
| 40 |
-
from
|
| 41 |
c["LLM routing, hand-written"] = [x for x in tasks.llm_route() if analyse(x[0])["language"] == "az"
|
| 42 |
or (analyse(x[0])["script"] == "latin" and not analyse(x[0])["is_english"])]
|
| 43 |
return c
|
|
@@ -60,10 +60,10 @@ def score(ag, trip):
|
|
| 60 |
|
| 61 |
if __name__ == "__main__":
|
| 62 |
C = cats()
|
| 63 |
-
models = {"base": ("<base-checkpoint-dir>", "multilingual"), "
|
| 64 |
res = defaultdict(dict)
|
| 65 |
for m, (p, sub) in models.items():
|
| 66 |
-
ag =
|
| 67 |
for name, trip in C.items():
|
| 68 |
if len(trip) < 15:
|
| 69 |
continue
|
|
|
|
| 3 |
from collections import defaultdict
|
| 4 |
import numpy as np
|
| 5 |
sys.path.insert(0, "."); sys.path.insert(0, "./bench")
|
| 6 |
+
import system01, tasks
|
| 7 |
+
from system01.common import ece_score
|
| 8 |
|
| 9 |
SYN = "./data/synth"
|
| 10 |
GUARD_NAMES = {"jailbreak": "Jailbreak detection", "prompt_injection": "Prompt-injection detection", "pii": "PII detection",
|
|
|
|
| 37 |
c["Voice-assistant domain (MASSIVE, 18 classes)"] = tasks.massive_scenario("az", "test", 1000)
|
| 38 |
c["App-review sentiment"] = tasks.az_sentiment("test", 1000)
|
| 39 |
c["News topic (SIB-200, zero-shot)"] = tasks.sib_any("az", "test")
|
| 40 |
+
from system01.lang import analyse
|
| 41 |
c["LLM routing, hand-written"] = [x for x in tasks.llm_route() if analyse(x[0])["language"] == "az"
|
| 42 |
or (analyse(x[0])["script"] == "latin" and not analyse(x[0])["is_english"])]
|
| 43 |
return c
|
|
|
|
| 60 |
|
| 61 |
if __name__ == "__main__":
|
| 62 |
C = cats()
|
| 63 |
+
models = {"base": ("<base-checkpoint-dir>", "multilingual"), "system01": (".", None)}
|
| 64 |
res = defaultdict(dict)
|
| 65 |
for m, (p, sub) in models.items():
|
| 66 |
+
ag = system01.load(p, subfolder=sub, device="cuda")
|
| 67 |
for name, trip in C.items():
|
| 68 |
if len(trip) < 15:
|
| 69 |
continue
|
bench/plots_v2.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
"""Charts for the
|
| 2 |
import json, sys
|
| 3 |
import numpy as np, matplotlib
|
| 4 |
matplotlib.use("Agg")
|
|
@@ -7,7 +7,7 @@ import matplotlib.pyplot as plt
|
|
| 7 |
RES = "./results/"
|
| 8 |
OUT = "./assets/"
|
| 9 |
SURF, INK, INK2, MUTED, GRID = "#fcfcfb", "#0b0b0b", "#52514e", "#8a8984", "#e6e5e0"
|
| 10 |
-
S1, CMP, THIRD = "#2a78d6", "#eb6834", "#1baf7a" #
|
| 11 |
plt.rcParams.update({"font.family": "DejaVu Sans", "font.size": 11, "axes.edgecolor": GRID, "axes.labelcolor": INK2,
|
| 12 |
"xtick.color": INK2, "ytick.color": INK2, "figure.facecolor": SURF, "axes.facecolor": SURF,
|
| 13 |
"savefig.facecolor": SURF, "axes.titleweight": "bold", "axes.titlesize": 13, "axes.titlecolor": INK})
|
|
@@ -16,7 +16,7 @@ AZC = json.load(open(RES + "az_categories.json"))
|
|
| 16 |
REL = json.load(open(RES + "reliability_v3.json"))
|
| 17 |
SPD = json.load(open(RES + "speed.json"))
|
| 18 |
DOOM = json.load(open(RES + "doom.json"))
|
| 19 |
-
M = "
|
| 20 |
|
| 21 |
|
| 22 |
def style(ax, axis="x"):
|
|
@@ -26,49 +26,50 @@ def style(ax, axis="x"):
|
|
| 26 |
|
| 27 |
|
| 28 |
def title(fig, t, sub):
|
| 29 |
-
|
| 30 |
-
fig.text(0.012, 0.
|
|
|
|
| 31 |
|
| 32 |
|
| 33 |
-
# 1.
|
| 34 |
jev = [("AG News\n4 topics", "AG News (4)", 0.910), ("SMS spam\nspam vs ham", "SMS spam (yes/no)", 0.930),
|
| 35 |
("DAIR Emotion\n6 emotions", "DAIR Emotion (6)", 0.480), ("Banking77\n77 intents", "Banking77 (77)", 0.763)]
|
| 36 |
fig, ax = plt.subplots(figsize=(11, 4.9)); fig.subplots_adjust(top=0.78, bottom=0.17, left=0.06, right=0.99)
|
| 37 |
x = np.arange(len(jev)); w = 0.36
|
| 38 |
a = [FINAL[k][M]["acc"] for _, k, _ in jev]; b = [v for *_, v in jev]
|
| 39 |
-
ax.bar(x - w / 2, a, w - 0.03, color=S1, label="
|
| 40 |
ax.bar(x + w / 2, b, w - 0.03, color=CMP, label="TypeSafe Jev 1.13.0 (third-party published)", edgecolor=SURF, linewidth=2)
|
| 41 |
for i in range(len(jev)):
|
| 42 |
ax.text(x[i] - w / 2, a[i] + 0.015, f"{a[i]:.3f}", ha="center", fontsize=10.5, color=INK, fontweight="bold")
|
| 43 |
ax.text(x[i] + w / 2, b[i] + 0.015, f"{b[i]:.3f}", ha="center", fontsize=10.5, color=INK2)
|
| 44 |
ax.set_xticks(x); ax.set_xticklabels([n for n, *_ in jev], color=INK); ax.set_ylim(0, 1.08); ax.set_ylabel("accuracy")
|
| 45 |
style(ax, "y"); ax.legend(loc="upper right", frameon=False, fontsize=9.5, labelcolor=INK2, ncol=1)
|
| 46 |
-
title(fig, "
|
| 47 |
"Zero-shot for both. Jev: AbdelStark pilot (n=100/set) and nibzard (Banking77, 77 labels). Latency: 9 ms vs 236–276 ms · ECE: 0.02–0.04 vs 0.246")
|
| 48 |
fig.savefig(OUT + "vs_jev.png", dpi=160); plt.close(fig)
|
| 49 |
|
| 50 |
# 2. Azerbaijani categories
|
| 51 |
order = sorted(AZC, key=lambda k: AZC[k][M]["acc"])
|
| 52 |
-
fig, ax = plt.subplots(figsize=(11, 0.46 * len(order) + 1.9)); fig.subplots_adjust(top=0.
|
| 53 |
y = np.arange(len(order)); h = 0.38
|
| 54 |
ax.barh(y - h / 2, [AZC[k]["base"]["acc"] for k in order], h - 0.04, color=MUTED, label="base checkpoint (before post-training)", edgecolor=SURF, linewidth=1.5)
|
| 55 |
-
ax.barh(y + h / 2, [AZC[k][M]["acc"] for k in order], h - 0.04, color=S1, label="
|
| 56 |
for i, k in enumerate(order):
|
| 57 |
ax.text(AZC[k][M]["acc"] + 0.008, i + h / 2, f"{AZC[k][M]['acc']:.2f}", va="center", fontsize=9.5, color=INK, fontweight="bold")
|
| 58 |
ax.set_yticks(y); ax.set_yticklabels([f"{k} (n={AZC[k][M]['n']})" for k in order], color=INK, fontsize=10)
|
| 59 |
ax.set_xlim(0, 1.06); ax.set_xlabel("accuracy on Azerbaijani (held-out)"); style(ax)
|
| 60 |
-
ax.legend(loc="lower
|
| 61 |
title(fig, "Azerbaijani, by task category", "Held-out schemas never seen in training, plus real Azerbaijani datasets (MASSIVE az-AZ, app reviews, SIB-200).")
|
| 62 |
fig.savefig(OUT + "azerbaijani_categories.png", dpi=160); plt.close(fig)
|
| 63 |
|
| 64 |
# 3. Multilingual: MASSIVE-scenario, 14 languages
|
| 65 |
langs = ["az", "en", "ru", "tr", "de", "fr", "es", "ar", "fa", "hi", "zh-CN", "ja", "ko", "ka"]
|
| 66 |
-
fig, ax = plt.subplots(figsize=(11, 4.4)); fig.subplots_adjust(top=0.
|
| 67 |
x = np.arange(len(langs))
|
| 68 |
bs = [FINAL[f"MASSIVE-scenario ({l})"]["base"]["acc"] for l in langs]; ss = [FINAL[f"MASSIVE-scenario ({l})"][M]["acc"] for l in langs]
|
| 69 |
ax.vlines(x, bs, ss, color=GRID, lw=3)
|
| 70 |
ax.scatter(x, bs, s=70, color=MUTED, zorder=3, label="base checkpoint", edgecolor=SURF, linewidth=2)
|
| 71 |
-
ax.scatter(x, ss, s=90, color=S1, zorder=3, label="
|
| 72 |
for i, v in enumerate(ss):
|
| 73 |
ax.text(i, v + 0.03, f"{v:.2f}", ha="center", fontsize=9, color=INK)
|
| 74 |
ax.set_xticks(x); ax.set_xticklabels(langs, color=INK); ax.set_ylim(0, 1.05); ax.set_ylabel("accuracy")
|
|
@@ -83,25 +84,32 @@ try:
|
|
| 83 |
except FileNotFoundError:
|
| 84 |
c4 = []
|
| 85 |
off = cur[-1]["step"] if cur else 0
|
| 86 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
for ax, key, lab in ((axs[0], "loss", "training loss (− proper-scoring reward)"), (axs[1], "acc", "training accuracy (argmax)")):
|
| 88 |
xs = [c["step"] for c in cur]; ys = [c[key] for c in cur]
|
| 89 |
k = 5; sm = np.convolve(ys, np.ones(k) / k, mode="valid")
|
| 90 |
ax.plot(xs, ys, color=S1, alpha=0.25, lw=1)
|
| 91 |
-
ax.plot(xs[k - 1:], sm, color=S1, lw=2, label="multilingual + Azerbaijani
|
| 92 |
if c4:
|
| 93 |
x4 = [off + c["step"] for c in c4]; y4 = [c[key] for c in c4]
|
| 94 |
-
ax.plot(x4, y4, color=CMP, lw=2, label="spatial
|
|
|
|
|
|
|
| 95 |
ep = next((c["step"] for c in cur if c.get("epoch") == 1), None)
|
| 96 |
if ep:
|
| 97 |
ax.axvline(ep, color=MUTED, ls=(0, (3, 3)), lw=1); ax.text(ep, ax.get_ylim()[1], " epoch 2", color=MUTED, fontsize=9, va="top")
|
| 98 |
ax.set_xlabel("optimizer step"); ax.set_title(lab, loc="left", fontsize=11.5); style(ax, "y")
|
| 99 |
axs[1].legend(loc="lower right", frameon=False, fontsize=9, labelcolor=INK2)
|
| 100 |
-
title(fig, "Post-training on one RTX 4090", "158k decisions, 18 languages, 2 epochs in 14.4 min
|
| 101 |
fig.savefig(OUT + "training.png", dpi=160); plt.close(fig)
|
| 102 |
|
| 103 |
# 5. Speed: single-request latency and batched throughput (two panels)
|
| 104 |
-
fig, axs = plt.subplots(1, 2, figsize=(11, 4.1)); fig.subplots_adjust(top=0.
|
| 105 |
bt = SPD["batched"]; ns = sorted(int(k) for k in bt)
|
| 106 |
axs[0].bar(range(len(ns)), [bt[str(n)]["total_ms"] for n in ns], color=S1, width=0.6, edgecolor=SURF, linewidth=2)
|
| 107 |
for i, n in enumerate(ns):
|
|
@@ -117,39 +125,39 @@ title(fig, f"Speed on one {SPD['gpu']}", f"Single request: {SPD['single_question
|
|
| 117 |
fig.savefig(OUT + "speed.png", dpi=160); plt.close(fig)
|
| 118 |
|
| 119 |
# 6. Calibration on Azerbaijani
|
| 120 |
-
fig, axs = plt.subplots(1, 2, figsize=(11, 4.5), gridspec_kw={"width_ratios": [1.2, 1]}); fig.subplots_adjust(top=0.
|
| 121 |
ax = axs[0]; ax.plot([0, 1], [0, 1], color=MUTED, lw=1, ls=(0, (4, 3)))
|
| 122 |
-
for m, c, lab in (("base", MUTED, "base"), ("
|
| 123 |
bn = REL[m]["bins"]
|
| 124 |
ax.plot([q["conf"] for q in bn], [q["acc"] for q in bn], color=c, lw=2, marker="o", ms=8, markeredgecolor=SURF, markeredgewidth=2)
|
| 125 |
-
ax.text(0.05, 0.9, f"
|
| 126 |
ax.plot([0.0, 0.035], [0.915, 0.915], color=S1, lw=3)
|
| 127 |
ax.text(0.05, 0.82, f"base ECE {REL['base']['ece']:.3f}", color=INK, fontsize=10.5)
|
| 128 |
ax.plot([0.0, 0.035], [0.835, 0.835], color=MUTED, lw=3)
|
| 129 |
ax.set_xlim(0, 1.02); ax.set_ylim(0, 1.02); ax.set_xlabel("reported confidence"); ax.set_ylabel("observed accuracy"); style(ax, "both")
|
| 130 |
-
ax.set_title(f"reliability, Azerbaijani (n={REL['
|
| 131 |
ax = axs[1]
|
| 132 |
-
vals = [("Jev (published)", 0.246, CMP), ("base", REL["base"]["ece"], MUTED), ("
|
| 133 |
ax.bar(range(3), [v for _, v, _ in vals], color=[c for *_, c in vals], width=0.55, edgecolor=SURF, linewidth=2)
|
| 134 |
for i, (_, v, _) in enumerate(vals):
|
| 135 |
ax.text(i, v + 0.006, f"{v:.3f}", ha="center", fontsize=10.5, color=INK, fontweight="bold" if i == 2 else None)
|
| 136 |
ax.set_xticks(range(3)); ax.set_xticklabels([n for n, *_ in vals], color=INK); ax.set_ylabel("ECE (lower is better)"); style(ax, "y")
|
| 137 |
ax.set_title("expected calibration error", loc="left", fontsize=11.5)
|
| 138 |
-
title(fig, "Calibrated probabilities", "When
|
| 139 |
fig.savefig(OUT + "calibration.png", dpi=160); plt.close(fig)
|
| 140 |
|
| 141 |
# 7. Doom
|
| 142 |
sc = ["defend_the_center", "defend_the_line", "deadly_corridor", "basic"]
|
| 143 |
-
fig, ax = plt.subplots(figsize=(11, 3.8)); fig.subplots_adjust(top=0.
|
| 144 |
x = np.arange(len(sc)); w = 0.36
|
| 145 |
-
ks = [DOOM[s]["
|
| 146 |
-
ax.bar(x - w / 2, ks, w - 0.03, color=S1, label="
|
| 147 |
ax.bar(x + w / 2, kr, w - 0.03, color=MUTED, label="random policy", edgecolor=SURF, linewidth=2)
|
| 148 |
for i in range(len(sc)):
|
| 149 |
ax.text(x[i] - w / 2, ks[i] + 0.25, f"{ks[i]:.1f}", ha="center", fontsize=10.5, color=INK, fontweight="bold")
|
| 150 |
ax.text(x[i] + w / 2, kr[i] + 0.25, f"{kr[i]:.1f}", ha="center", fontsize=10, color=INK2)
|
| 151 |
ax.set_xticks(x); ax.set_xticklabels(sc, color=INK); ax.set_ylabel("kills per episode"); style(ax, "y")
|
| 152 |
ax.legend(frameon=False, fontsize=9.5, labelcolor=INK2)
|
| 153 |
-
title(fig, "Doom (ViZDoom),
|
| 154 |
fig.savefig(OUT + "doom_results.png", dpi=160); plt.close(fig)
|
| 155 |
print("ok")
|
|
|
|
| 1 |
+
"""Charts for the System01 model card (light surface, validated 3-slot palette)."""
|
| 2 |
import json, sys
|
| 3 |
import numpy as np, matplotlib
|
| 4 |
matplotlib.use("Agg")
|
|
|
|
| 7 |
RES = "./results/"
|
| 8 |
OUT = "./assets/"
|
| 9 |
SURF, INK, INK2, MUTED, GRID = "#fcfcfb", "#0b0b0b", "#52514e", "#8a8984", "#e6e5e0"
|
| 10 |
+
S1, CMP, THIRD = "#2a78d6", "#eb6834", "#1baf7a" # System01, comparison, third series
|
| 11 |
plt.rcParams.update({"font.family": "DejaVu Sans", "font.size": 11, "axes.edgecolor": GRID, "axes.labelcolor": INK2,
|
| 12 |
"xtick.color": INK2, "ytick.color": INK2, "figure.facecolor": SURF, "axes.facecolor": SURF,
|
| 13 |
"savefig.facecolor": SURF, "axes.titleweight": "bold", "axes.titlesize": 13, "axes.titlecolor": INK})
|
|
|
|
| 16 |
REL = json.load(open(RES + "reliability_v3.json"))
|
| 17 |
SPD = json.load(open(RES + "speed.json"))
|
| 18 |
DOOM = json.load(open(RES + "doom.json"))
|
| 19 |
+
M = "system01"
|
| 20 |
|
| 21 |
|
| 22 |
def style(ax, axis="x"):
|
|
|
|
| 26 |
|
| 27 |
|
| 28 |
def title(fig, t, sub):
|
| 29 |
+
H = fig.get_figheight()
|
| 30 |
+
fig.text(0.012, 1 - 0.30 / H, t, ha="left", va="center", fontsize=15, fontweight="bold", color=INK)
|
| 31 |
+
fig.text(0.012, 1 - 0.62 / H, sub, ha="left", va="center", fontsize=10.5, color=INK2)
|
| 32 |
|
| 33 |
|
| 34 |
+
# 1. System01 vs Jev on the public sets Jev was published on
|
| 35 |
jev = [("AG News\n4 topics", "AG News (4)", 0.910), ("SMS spam\nspam vs ham", "SMS spam (yes/no)", 0.930),
|
| 36 |
("DAIR Emotion\n6 emotions", "DAIR Emotion (6)", 0.480), ("Banking77\n77 intents", "Banking77 (77)", 0.763)]
|
| 37 |
fig, ax = plt.subplots(figsize=(11, 4.9)); fig.subplots_adjust(top=0.78, bottom=0.17, left=0.06, right=0.99)
|
| 38 |
x = np.arange(len(jev)); w = 0.36
|
| 39 |
a = [FINAL[k][M]["acc"] for _, k, _ in jev]; b = [v for *_, v in jev]
|
| 40 |
+
ax.bar(x - w / 2, a, w - 0.03, color=S1, label="System01 (measured, n≈1,000 each)", edgecolor=SURF, linewidth=2)
|
| 41 |
ax.bar(x + w / 2, b, w - 0.03, color=CMP, label="TypeSafe Jev 1.13.0 (third-party published)", edgecolor=SURF, linewidth=2)
|
| 42 |
for i in range(len(jev)):
|
| 43 |
ax.text(x[i] - w / 2, a[i] + 0.015, f"{a[i]:.3f}", ha="center", fontsize=10.5, color=INK, fontweight="bold")
|
| 44 |
ax.text(x[i] + w / 2, b[i] + 0.015, f"{b[i]:.3f}", ha="center", fontsize=10.5, color=INK2)
|
| 45 |
ax.set_xticks(x); ax.set_xticklabels([n for n, *_ in jev], color=INK); ax.set_ylim(0, 1.08); ax.set_ylabel("accuracy")
|
| 46 |
style(ax, "y"); ax.legend(loc="upper right", frameon=False, fontsize=9.5, labelcolor=INK2, ncol=1)
|
| 47 |
+
title(fig, "System01 vs TypeSafe Jev on the same public datasets",
|
| 48 |
"Zero-shot for both. Jev: AbdelStark pilot (n=100/set) and nibzard (Banking77, 77 labels). Latency: 9 ms vs 236–276 ms · ECE: 0.02–0.04 vs 0.246")
|
| 49 |
fig.savefig(OUT + "vs_jev.png", dpi=160); plt.close(fig)
|
| 50 |
|
| 51 |
# 2. Azerbaijani categories
|
| 52 |
order = sorted(AZC, key=lambda k: AZC[k][M]["acc"])
|
| 53 |
+
fig, ax = plt.subplots(figsize=(11, 0.46 * len(order) + 1.9)); fig.subplots_adjust(top=0.84, left=0.36, right=0.97, bottom=0.07)
|
| 54 |
y = np.arange(len(order)); h = 0.38
|
| 55 |
ax.barh(y - h / 2, [AZC[k]["base"]["acc"] for k in order], h - 0.04, color=MUTED, label="base checkpoint (before post-training)", edgecolor=SURF, linewidth=1.5)
|
| 56 |
+
ax.barh(y + h / 2, [AZC[k][M]["acc"] for k in order], h - 0.04, color=S1, label="System01", edgecolor=SURF, linewidth=1.5)
|
| 57 |
for i, k in enumerate(order):
|
| 58 |
ax.text(AZC[k][M]["acc"] + 0.008, i + h / 2, f"{AZC[k][M]['acc']:.2f}", va="center", fontsize=9.5, color=INK, fontweight="bold")
|
| 59 |
ax.set_yticks(y); ax.set_yticklabels([f"{k} (n={AZC[k][M]['n']})" for k in order], color=INK, fontsize=10)
|
| 60 |
ax.set_xlim(0, 1.06); ax.set_xlabel("accuracy on Azerbaijani (held-out)"); style(ax)
|
| 61 |
+
ax.legend(loc="lower left", bbox_to_anchor=(0, 1.0), ncol=2, frameon=False, fontsize=9.5, labelcolor=INK2)
|
| 62 |
title(fig, "Azerbaijani, by task category", "Held-out schemas never seen in training, plus real Azerbaijani datasets (MASSIVE az-AZ, app reviews, SIB-200).")
|
| 63 |
fig.savefig(OUT + "azerbaijani_categories.png", dpi=160); plt.close(fig)
|
| 64 |
|
| 65 |
# 3. Multilingual: MASSIVE-scenario, 14 languages
|
| 66 |
langs = ["az", "en", "ru", "tr", "de", "fr", "es", "ar", "fa", "hi", "zh-CN", "ja", "ko", "ka"]
|
| 67 |
+
fig, ax = plt.subplots(figsize=(11, 4.4)); fig.subplots_adjust(top=0.76, bottom=0.12, left=0.07, right=0.99)
|
| 68 |
x = np.arange(len(langs))
|
| 69 |
bs = [FINAL[f"MASSIVE-scenario ({l})"]["base"]["acc"] for l in langs]; ss = [FINAL[f"MASSIVE-scenario ({l})"][M]["acc"] for l in langs]
|
| 70 |
ax.vlines(x, bs, ss, color=GRID, lw=3)
|
| 71 |
ax.scatter(x, bs, s=70, color=MUTED, zorder=3, label="base checkpoint", edgecolor=SURF, linewidth=2)
|
| 72 |
+
ax.scatter(x, ss, s=90, color=S1, zorder=3, label="System01", edgecolor=SURF, linewidth=2)
|
| 73 |
for i, v in enumerate(ss):
|
| 74 |
ax.text(i, v + 0.03, f"{v:.2f}", ha="center", fontsize=9, color=INK)
|
| 75 |
ax.set_xticks(x); ax.set_xticklabels(langs, color=INK); ax.set_ylim(0, 1.05); ax.set_ylabel("accuracy")
|
|
|
|
| 84 |
except FileNotFoundError:
|
| 85 |
c4 = []
|
| 86 |
off = cur[-1]["step"] if cur else 0
|
| 87 |
+
try:
|
| 88 |
+
c5 = [json.loads(l) for l in open(RES + "train_curve_v5.jsonl")]
|
| 89 |
+
except FileNotFoundError:
|
| 90 |
+
c5 = []
|
| 91 |
+
off5 = off + (c4[-1]["step"] if c4 else 0)
|
| 92 |
+
fig, axs = plt.subplots(1, 2, figsize=(11, 3.9)); fig.subplots_adjust(top=0.71, bottom=0.15, left=0.07, right=0.99, wspace=0.22)
|
| 93 |
for ax, key, lab in ((axs[0], "loss", "training loss (− proper-scoring reward)"), (axs[1], "acc", "training accuracy (argmax)")):
|
| 94 |
xs = [c["step"] for c in cur]; ys = [c[key] for c in cur]
|
| 95 |
k = 5; sm = np.convolve(ys, np.ones(k) / k, mode="valid")
|
| 96 |
ax.plot(xs, ys, color=S1, alpha=0.25, lw=1)
|
| 97 |
+
ax.plot(xs[k - 1:], sm, color=S1, lw=2, label="stage 1 · multilingual + Azerbaijani")
|
| 98 |
if c4:
|
| 99 |
x4 = [off + c["step"] for c in c4]; y4 = [c[key] for c in c4]
|
| 100 |
+
ax.plot(x4, y4, color=CMP, lw=2, label="stage 2 · spatial control")
|
| 101 |
+
if c5:
|
| 102 |
+
ax.plot([off5 + c["step"] for c in c5], [c[key] for c in c5], color=THIRD, lw=2, label="stage 3 · routing emphasis")
|
| 103 |
ep = next((c["step"] for c in cur if c.get("epoch") == 1), None)
|
| 104 |
if ep:
|
| 105 |
ax.axvline(ep, color=MUTED, ls=(0, (3, 3)), lw=1); ax.text(ep, ax.get_ylim()[1], " epoch 2", color=MUTED, fontsize=9, va="top")
|
| 106 |
ax.set_xlabel("optimizer step"); ax.set_title(lab, loc="left", fontsize=11.5); style(ax, "y")
|
| 107 |
axs[1].legend(loc="lower right", frameon=False, fontsize=9, labelcolor=INK2)
|
| 108 |
+
title(fig, "Post-training on one RTX 4090", "Stage 1: 158k decisions, 18 languages, 2 epochs in 14.4 min. Stages 2–3: 2 min each. Objective: RLCD proper-scoring reward.")
|
| 109 |
fig.savefig(OUT + "training.png", dpi=160); plt.close(fig)
|
| 110 |
|
| 111 |
# 5. Speed: single-request latency and batched throughput (two panels)
|
| 112 |
+
fig, axs = plt.subplots(1, 2, figsize=(11, 4.1)); fig.subplots_adjust(top=0.72, bottom=0.16, left=0.07, right=0.98, wspace=0.25)
|
| 113 |
bt = SPD["batched"]; ns = sorted(int(k) for k in bt)
|
| 114 |
axs[0].bar(range(len(ns)), [bt[str(n)]["total_ms"] for n in ns], color=S1, width=0.6, edgecolor=SURF, linewidth=2)
|
| 115 |
for i, n in enumerate(ns):
|
|
|
|
| 125 |
fig.savefig(OUT + "speed.png", dpi=160); plt.close(fig)
|
| 126 |
|
| 127 |
# 6. Calibration on Azerbaijani
|
| 128 |
+
fig, axs = plt.subplots(1, 2, figsize=(11, 4.5), gridspec_kw={"width_ratios": [1.2, 1]}); fig.subplots_adjust(top=0.74, bottom=0.13, left=0.07, right=0.98, wspace=0.28)
|
| 129 |
ax = axs[0]; ax.plot([0, 1], [0, 1], color=MUTED, lw=1, ls=(0, (4, 3)))
|
| 130 |
+
for m, c, lab in (("base", MUTED, "base"), ("system01", S1, "System01")):
|
| 131 |
bn = REL[m]["bins"]
|
| 132 |
ax.plot([q["conf"] for q in bn], [q["acc"] for q in bn], color=c, lw=2, marker="o", ms=8, markeredgecolor=SURF, markeredgewidth=2)
|
| 133 |
+
ax.text(0.05, 0.9, f"System01 ECE {REL['system01']['ece']:.3f}", color=INK, fontsize=10.5)
|
| 134 |
ax.plot([0.0, 0.035], [0.915, 0.915], color=S1, lw=3)
|
| 135 |
ax.text(0.05, 0.82, f"base ECE {REL['base']['ece']:.3f}", color=INK, fontsize=10.5)
|
| 136 |
ax.plot([0.0, 0.035], [0.835, 0.835], color=MUTED, lw=3)
|
| 137 |
ax.set_xlim(0, 1.02); ax.set_ylim(0, 1.02); ax.set_xlabel("reported confidence"); ax.set_ylabel("observed accuracy"); style(ax, "both")
|
| 138 |
+
ax.set_title(f"reliability, Azerbaijani (n={REL['system01']['n']:,})", loc="left", fontsize=11.5)
|
| 139 |
ax = axs[1]
|
| 140 |
+
vals = [("Jev (published)", 0.246, CMP), ("base", REL["base"]["ece"], MUTED), ("System01", REL["system01"]["ece"], S1)]
|
| 141 |
ax.bar(range(3), [v for _, v, _ in vals], color=[c for *_, c in vals], width=0.55, edgecolor=SURF, linewidth=2)
|
| 142 |
for i, (_, v, _) in enumerate(vals):
|
| 143 |
ax.text(i, v + 0.006, f"{v:.3f}", ha="center", fontsize=10.5, color=INK, fontweight="bold" if i == 2 else None)
|
| 144 |
ax.set_xticks(range(3)); ax.set_xticklabels([n for n, *_ in vals], color=INK); ax.set_ylabel("ECE (lower is better)"); style(ax, "y")
|
| 145 |
ax.set_title("expected calibration error", loc="left", fontsize=11.5)
|
| 146 |
+
title(fig, "Calibrated probabilities", "When System01 says 90%, it is right about 90% of the time — so confidence thresholds are safe to automate on.")
|
| 147 |
fig.savefig(OUT + "calibration.png", dpi=160); plt.close(fig)
|
| 148 |
|
| 149 |
# 7. Doom
|
| 150 |
sc = ["defend_the_center", "defend_the_line", "deadly_corridor", "basic"]
|
| 151 |
+
fig, ax = plt.subplots(figsize=(11, 3.8)); fig.subplots_adjust(top=0.72, bottom=0.14, left=0.07, right=0.99)
|
| 152 |
x = np.arange(len(sc)); w = 0.36
|
| 153 |
+
ks = [DOOM[s]["system01"]["kills_mean"] for s in sc]; kr = [DOOM[s]["random"]["kills_mean"] for s in sc]
|
| 154 |
+
ax.bar(x - w / 2, ks, w - 0.03, color=S1, label="System01 (zero-shot)", edgecolor=SURF, linewidth=2)
|
| 155 |
ax.bar(x + w / 2, kr, w - 0.03, color=MUTED, label="random policy", edgecolor=SURF, linewidth=2)
|
| 156 |
for i in range(len(sc)):
|
| 157 |
ax.text(x[i] - w / 2, ks[i] + 0.25, f"{ks[i]:.1f}", ha="center", fontsize=10.5, color=INK, fontweight="bold")
|
| 158 |
ax.text(x[i] + w / 2, kr[i] + 0.25, f"{kr[i]:.1f}", ha="center", fontsize=10, color=INK2)
|
| 159 |
ax.set_xticks(x); ax.set_xticklabels(sc, color=INK); ax.set_ylabel("kills per episode"); style(ax, "y")
|
| 160 |
ax.legend(frameon=False, fontsize=9.5, labelcolor=INK2)
|
| 161 |
+
title(fig, "Doom (ViZDoom), 20 episodes per scenario", "Two typed questions per step, one forward pass, ~10 ms per decision. No Doom data in training.")
|
| 162 |
fig.savefig(OUT + "doom_results.png", dpi=160); plt.close(fig)
|
| 163 |
print("ok")
|
bench/run_bench.py
CHANGED
|
@@ -2,12 +2,12 @@
|
|
| 2 |
import argparse, json, sys, time
|
| 3 |
import numpy as np, torch
|
| 4 |
sys.path.insert(0, "./pkg"); sys.path.insert(0, "./bench")
|
| 5 |
-
import
|
| 6 |
-
from
|
| 7 |
-
from
|
| 8 |
|
| 9 |
ap = argparse.ArgumentParser()
|
| 10 |
-
ap.add_argument("--models", default="
|
| 11 |
ap.add_argument("--n", type=int, default=1000)
|
| 12 |
ap.add_argument("--out", default="results/bench.json")
|
| 13 |
ap.add_argument("--router", default="") # e.g. "en=english,other=multilingual,az=azerbaijani"
|
|
|
|
| 2 |
import argparse, json, sys, time
|
| 3 |
import numpy as np, torch
|
| 4 |
sys.path.insert(0, "./pkg"); sys.path.insert(0, "./bench")
|
| 5 |
+
import system01 as systemone, tasks
|
| 6 |
+
from system01.lang import analyse
|
| 7 |
+
from system01.common import ece_score
|
| 8 |
|
| 9 |
ap = argparse.ArgumentParser()
|
| 10 |
+
ap.add_argument("--models", default="system01=.")
|
| 11 |
ap.add_argument("--n", type=int, default=1000)
|
| 12 |
ap.add_argument("--out", default="results/bench.json")
|
| 13 |
ap.add_argument("--router", default="") # e.g. "en=english,other=multilingual,az=azerbaijani"
|
bench/speed.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
-
"""Speed of
|
| 2 |
|
| 3 |
-
python bench/speed.py --model ckpt/
|
| 4 |
"""
|
| 5 |
import argparse, asyncio, json, sys, time
|
| 6 |
import numpy as np, torch
|
|
@@ -8,7 +8,7 @@ sys.path.insert(0, "./pkg")
|
|
| 8 |
import systemone
|
| 9 |
|
| 10 |
ap = argparse.ArgumentParser()
|
| 11 |
-
ap.add_argument("--model", default="ckpt/
|
| 12 |
ap.add_argument("--out", default="results/speed.json")
|
| 13 |
ap.add_argument("--api", default="")
|
| 14 |
ap.add_argument("--key", default="")
|
|
@@ -60,7 +60,7 @@ if a.api:
|
|
| 60 |
|
| 61 |
async def http():
|
| 62 |
async with httpx.AsyncClient(timeout=60) as c:
|
| 63 |
-
body = lambda i: {"state": STATES[i % len(STATES)], "questions": Q, "model": "
|
| 64 |
await c.post(a.api + "/v1/decide", json=body(0), headers=H)
|
| 65 |
t0 = time.perf_counter()
|
| 66 |
rs = await asyncio.gather(*(c.post(a.api + "/v1/decide", json=body(i), headers=H) for i in range(150)))
|
|
@@ -68,7 +68,7 @@ if a.api:
|
|
| 68 |
ok = sum(r.status_code == 200 for r in rs)
|
| 69 |
t0 = time.perf_counter()
|
| 70 |
r = await c.post(a.api + "/v1/decide/batch", json={"items": [{"state": STATES[i % 6], "questions": Q} for i in range(150)],
|
| 71 |
-
"model": "
|
| 72 |
batch = (time.perf_counter() - t0) * 1000
|
| 73 |
return conc, ok, batch, r.status_code
|
| 74 |
conc, ok, batch, code = asyncio.run(http())
|
|
|
|
| 1 |
+
"""Speed of System01 on one GPU: single-request latency, batched throughput, and HTTP concurrency.
|
| 2 |
|
| 3 |
+
python bench/speed.py --model ckpt/system01 --out results/speed.json [--api http://localhost:8095 --key ...]
|
| 4 |
"""
|
| 5 |
import argparse, asyncio, json, sys, time
|
| 6 |
import numpy as np, torch
|
|
|
|
| 8 |
import systemone
|
| 9 |
|
| 10 |
ap = argparse.ArgumentParser()
|
| 11 |
+
ap.add_argument("--model", default="ckpt/system01")
|
| 12 |
ap.add_argument("--out", default="results/speed.json")
|
| 13 |
ap.add_argument("--api", default="")
|
| 14 |
ap.add_argument("--key", default="")
|
|
|
|
| 60 |
|
| 61 |
async def http():
|
| 62 |
async with httpx.AsyncClient(timeout=60) as c:
|
| 63 |
+
body = lambda i: {"state": STATES[i % len(STATES)], "questions": Q, "model": "system01"}
|
| 64 |
await c.post(a.api + "/v1/decide", json=body(0), headers=H)
|
| 65 |
t0 = time.perf_counter()
|
| 66 |
rs = await asyncio.gather(*(c.post(a.api + "/v1/decide", json=body(i), headers=H) for i in range(150)))
|
|
|
|
| 68 |
ok = sum(r.status_code == 200 for r in rs)
|
| 69 |
t0 = time.perf_counter()
|
| 70 |
r = await c.post(a.api + "/v1/decide/batch", json={"items": [{"state": STATES[i % 6], "questions": Q} for i in range(150)],
|
| 71 |
+
"model": "system01"}, headers=H)
|
| 72 |
batch = (time.perf_counter() - t0) * 1000
|
| 73 |
return conc, ok, batch, r.status_code
|
| 74 |
conc, ok, batch, code = asyncio.run(http())
|
config.json
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_type": "system01",
|
| 3 |
+
"architectures": [
|
| 4 |
+
"System01DecisionModel"
|
| 5 |
+
],
|
| 6 |
+
"library": "system01",
|
| 7 |
+
"encoder": "jhu-clsp/mmBERT-base",
|
| 8 |
+
"head_layers": 2,
|
| 9 |
+
"max_len": 1024,
|
| 10 |
+
"answer_types": [
|
| 11 |
+
"choice",
|
| 12 |
+
"noul",
|
| 13 |
+
"score"
|
| 14 |
+
],
|
| 15 |
+
"decision_config": "rl_agent_config.json"
|
| 16 |
+
}
|
doom/bench_doom.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
-
"""Headless Doom benchmark + GIF recorder for
|
| 2 |
|
| 3 |
-
python doom/bench_doom.py --model doofz/
|
| 4 |
-
python doom/bench_doom.py --model doofz/
|
| 5 |
"""
|
| 6 |
import argparse, json, os, random, sys, time
|
| 7 |
from collections import Counter, deque
|
|
@@ -34,7 +34,7 @@ def compose(screen, ev):
|
|
| 34 |
img.paste(Image.fromarray(screen), (0, 0))
|
| 35 |
d = ImageDraw.Draw(img)
|
| 36 |
x = 656
|
| 37 |
-
d.text((x, 12), "
|
| 38 |
d.text((x, 42), "no vision · no text generation · 1 forward pass", font=F["s"], fill=(150, 148, 140))
|
| 39 |
d.text((x, 74), "CHOICE · next action", font=F["h"], fill=(120, 170, 240))
|
| 40 |
y = 98
|
|
@@ -54,6 +54,8 @@ def compose(screen, ev):
|
|
| 54 |
d.rounded_rectangle((x, y + 2, x + 318, y + 16), 7, fill=(43, 43, 41))
|
| 55 |
d.rounded_rectangle((x, y + 2, x + max(6, int(318 * t)), y + 16), 7, fill=(230, 103, 103))
|
| 56 |
d.text((x + 326, y + 1), f"{100 * t:3.0f}%", font=F["m"], fill=(200, 198, 190)); y += 30
|
|
|
|
|
|
|
| 57 |
d.text((x, y), f"{ev['latency_ms']:.1f} ms / decision · health {ev['hp']:.0f} · kills {ev['kills']:.0f}",
|
| 58 |
font=F["b"], fill=(244, 243, 238)); y += 30
|
| 59 |
d.text((x, y), "STATE · what the model reads", font=F["h"], fill=(120, 170, 240)); y += 22
|
|
@@ -68,7 +70,7 @@ def compose(screen, ev):
|
|
| 68 |
|
| 69 |
def main():
|
| 70 |
ap = argparse.ArgumentParser()
|
| 71 |
-
ap.add_argument("--model", default="doofz/
|
| 72 |
ap.add_argument("--scenarios", default="defend_the_center,defend_the_line,basic,deadly_corridor")
|
| 73 |
ap.add_argument("--episodes", type=int, default=10)
|
| 74 |
ap.add_argument("--out", default="")
|
|
@@ -101,7 +103,7 @@ def main():
|
|
| 101 |
|
| 102 |
res = {}
|
| 103 |
for scen in a.scenarios.split(","):
|
| 104 |
-
for policy in ("
|
| 105 |
g = D.Game(agent, scen, a.skill, 4, realtime=False)
|
| 106 |
game = g._make_game(vzd)
|
| 107 |
buttons = [b.name for b in game.get_available_buttons()]
|
|
@@ -120,7 +122,7 @@ def main():
|
|
| 120 |
me = next((o for o in st.objects if o.name == "DoomPlayer"), None)
|
| 121 |
pos = (round(me.position_x), round(me.position_y)) if me else None
|
| 122 |
stuck = bool(pos and pos == last and hist and hist[-1].startswith("MOVE")); last = pos
|
| 123 |
-
if policy == "
|
| 124 |
text, _, _ = D.describe(game, st, list(hist), stuck, vzd)
|
| 125 |
t0 = time.perf_counter()
|
| 126 |
act = D.policy(agent.predict(text, q)["answers"], crit)
|
|
|
|
| 1 |
+
"""Headless Doom benchmark + GIF recorder for System01.
|
| 2 |
|
| 3 |
+
python doom/bench_doom.py --model doofz/System01 --episodes 10 --out results/doom.json
|
| 4 |
+
python doom/bench_doom.py --model doofz/System01 --gif release/assets/doom.gif --gif-seconds 20
|
| 5 |
"""
|
| 6 |
import argparse, json, os, random, sys, time
|
| 7 |
from collections import Counter, deque
|
|
|
|
| 34 |
img.paste(Image.fromarray(screen), (0, 0))
|
| 35 |
d = ImageDraw.Draw(img)
|
| 36 |
x = 656
|
| 37 |
+
d.text((x, 12), "System01 plays Doom", font=F["t"], fill=(244, 243, 238))
|
| 38 |
d.text((x, 42), "no vision · no text generation · 1 forward pass", font=F["s"], fill=(150, 148, 140))
|
| 39 |
d.text((x, 74), "CHOICE · next action", font=F["h"], fill=(120, 170, 240))
|
| 40 |
y = 98
|
|
|
|
| 54 |
d.rounded_rectangle((x, y + 2, x + 318, y + 16), 7, fill=(43, 43, 41))
|
| 55 |
d.rounded_rectangle((x, y + 2, x + max(6, int(318 * t)), y + 16), 7, fill=(230, 103, 103))
|
| 56 |
d.text((x + 326, y + 1), f"{100 * t:3.0f}%", font=F["m"], fill=(200, 198, 190)); y += 30
|
| 57 |
+
why = "noul ≥ 0.5 → fire" if ev["action"] == "ATTACK" and ev["threat"] >= 0.5 else "top choice"
|
| 58 |
+
d.text((x, y), f"KEY PRESSED: {ev['action']} ({why})", font=F["b"], fill=(120, 200, 150)); y += 26
|
| 59 |
d.text((x, y), f"{ev['latency_ms']:.1f} ms / decision · health {ev['hp']:.0f} · kills {ev['kills']:.0f}",
|
| 60 |
font=F["b"], fill=(244, 243, 238)); y += 30
|
| 61 |
d.text((x, y), "STATE · what the model reads", font=F["h"], fill=(120, 170, 240)); y += 22
|
|
|
|
| 70 |
|
| 71 |
def main():
|
| 72 |
ap = argparse.ArgumentParser()
|
| 73 |
+
ap.add_argument("--model", default="doofz/System01")
|
| 74 |
ap.add_argument("--scenarios", default="defend_the_center,defend_the_line,basic,deadly_corridor")
|
| 75 |
ap.add_argument("--episodes", type=int, default=10)
|
| 76 |
ap.add_argument("--out", default="")
|
|
|
|
| 103 |
|
| 104 |
res = {}
|
| 105 |
for scen in a.scenarios.split(","):
|
| 106 |
+
for policy in ("system01", "random"):
|
| 107 |
g = D.Game(agent, scen, a.skill, 4, realtime=False)
|
| 108 |
game = g._make_game(vzd)
|
| 109 |
buttons = [b.name for b in game.get_available_buttons()]
|
|
|
|
| 122 |
me = next((o for o in st.objects if o.name == "DoomPlayer"), None)
|
| 123 |
pos = (round(me.position_x), round(me.position_y)) if me else None
|
| 124 |
stuck = bool(pos and pos == last and hist and hist[-1].startswith("MOVE")); last = pos
|
| 125 |
+
if policy == "system01":
|
| 126 |
text, _, _ = D.describe(game, st, list(hist), stuck, vzd)
|
| 127 |
t0 = time.perf_counter()
|
| 128 |
act = D.policy(agent.predict(text, q)["answers"], crit)
|
doom/doom_live.py
CHANGED
|
@@ -1,13 +1,13 @@
|
|
| 1 |
-
"""
|
| 2 |
|
| 3 |
Every few game tics the engine state (health, ammo, enemies with bearing / distance / visibility,
|
| 4 |
-
items, depth ahead, recent actions) is written as text.
|
| 5 |
ONE forward pass:
|
| 6 |
choice -> which of 7 actions to press
|
| 7 |
noul -> is an enemy inside the crosshair
|
| 8 |
The browser shows the game on the left and the model's decisions on the right.
|
| 9 |
|
| 10 |
-
python doom/doom_live.py --model doofz/
|
| 11 |
"""
|
| 12 |
import argparse, io, json, math, os, queue, sys, threading, time
|
| 13 |
from collections import Counter, deque
|
|
@@ -252,13 +252,13 @@ def serve(g, port):
|
|
| 252 |
|
| 253 |
def load_agent(path):
|
| 254 |
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
|
| 255 |
-
import
|
| 256 |
-
return
|
| 257 |
|
| 258 |
|
| 259 |
if __name__ == "__main__":
|
| 260 |
ap = argparse.ArgumentParser()
|
| 261 |
-
ap.add_argument("--model", default="doofz/
|
| 262 |
ap.add_argument("--scenario", default="defend_the_center", choices=sorted(SCENARIOS))
|
| 263 |
ap.add_argument("--skill", type=int, default=2)
|
| 264 |
ap.add_argument("--tics", type=int, default=4)
|
|
|
|
| 1 |
+
"""System01 plays Doom: no vision, no text generation.
|
| 2 |
|
| 3 |
Every few game tics the engine state (health, ammo, enemies with bearing / distance / visibility,
|
| 4 |
+
items, depth ahead, recent actions) is written as text. System01 answers two typed questions in
|
| 5 |
ONE forward pass:
|
| 6 |
choice -> which of 7 actions to press
|
| 7 |
noul -> is an enemy inside the crosshair
|
| 8 |
The browser shows the game on the left and the model's decisions on the right.
|
| 9 |
|
| 10 |
+
python doom/doom_live.py --model doofz/System01 --port 8096
|
| 11 |
"""
|
| 12 |
import argparse, io, json, math, os, queue, sys, threading, time
|
| 13 |
from collections import Counter, deque
|
|
|
|
| 252 |
|
| 253 |
def load_agent(path):
|
| 254 |
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
|
| 255 |
+
import system01
|
| 256 |
+
return system01.load(path, device="cuda")
|
| 257 |
|
| 258 |
|
| 259 |
if __name__ == "__main__":
|
| 260 |
ap = argparse.ArgumentParser()
|
| 261 |
+
ap.add_argument("--model", default="doofz/System01")
|
| 262 |
ap.add_argument("--scenario", default="defend_the_center", choices=sorted(SCENARIOS))
|
| 263 |
ap.add_argument("--skill", type=int, default=2)
|
| 264 |
ap.add_argument("--tics", type=int, default=4)
|
doom/index.html
CHANGED
|
@@ -3,7 +3,7 @@
|
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8">
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1">
|
| 6 |
-
<title>
|
| 7 |
<style>
|
| 8 |
:root {
|
| 9 |
--bg: #0e0e0d; --panel: #171716; --line: #2a2a28; --ink: #f4f3ee; --ink2: #b7b5ab; --muted: #7d7b73;
|
|
@@ -22,7 +22,7 @@
|
|
| 22 |
.game { background: #000; border: 1px solid var(--line); border-radius: 10px; overflow: hidden; position: relative; }
|
| 23 |
.game img { width: 100%; display: block; image-rendering: pixelated; aspect-ratio: 4/3; }
|
| 24 |
.tag { position: absolute; left: 12px; top: 10px; background: rgba(0,0,0,.6); padding: 3px 8px; border-radius: 5px; font-size: 12px; color: var(--ink2); }
|
| 25 |
-
.row { display: grid; grid-template-columns: repeat(
|
| 26 |
.stat { background: var(--panel); border: 1px solid var(--line); border-radius: 8px; padding: 10px 12px; }
|
| 27 |
.stat .k { color: var(--muted); font-size: 11px; text-transform: uppercase; letter-spacing: .6px; }
|
| 28 |
.stat .v { font-size: 22px; font-weight: 650; font-variant-numeric: tabular-nums; }
|
|
@@ -51,15 +51,16 @@
|
|
| 51 |
</head>
|
| 52 |
<body>
|
| 53 |
<header>
|
| 54 |
-
<h1>
|
| 55 |
<span class="sub">no vision · no text generation · two typed questions answered in one forward pass per step</span>
|
| 56 |
<span class="sp"></span>
|
| 57 |
<span class="scen" id="scen"></span>
|
| 58 |
</header>
|
| 59 |
<main>
|
| 60 |
<section>
|
| 61 |
-
<div class="game"><img src="/stream.mjpg" alt="Live Doom game played by
|
| 62 |
<div class="row">
|
|
|
|
| 63 |
<div class="stat"><div class="k">Latency / decision</div><div class="v" id="lat">–</div></div>
|
| 64 |
<div class="stat"><div class="k">Decisions / s</div><div class="v" id="dps">–</div></div>
|
| 65 |
<div class="stat"><div class="k">Health</div><div class="v" id="hp">–</div></div>
|
|
@@ -89,7 +90,7 @@
|
|
| 89 |
</div>
|
| 90 |
</aside>
|
| 91 |
</main>
|
| 92 |
-
<footer>
|
| 93 |
<script>
|
| 94 |
const ORDER = ["ATTACK","TURN_LEFT","TURN_RIGHT","MOVE_FORWARD","MOVE_LEFT","MOVE_RIGHT","MOVE_BACKWARD"];
|
| 95 |
const SCEN = ["defend_the_center","deadly_corridor","defend_the_line","basic"];
|
|
@@ -112,11 +113,13 @@ es.onmessage = m => {
|
|
| 112 |
if (e.type === "episode") { episodes(e.episodes); return; }
|
| 113 |
if (e.scenario !== current) { current = e.scenario; scenButtons(); $("scenTag").textContent = current; }
|
| 114 |
$("lat").textContent = e.latency_ms.toFixed(1) + " ms";
|
|
|
|
| 115 |
$("dps").textContent = e.dps.toFixed(0);
|
| 116 |
$("hp").textContent = Math.round(e.hp);
|
| 117 |
$("kills").textContent = Math.round(e.kills);
|
| 118 |
$("state").textContent = e.state;
|
| 119 |
-
|
|
|
|
| 120 |
$("thr").style.width = (100 * e.threat).toFixed(1) + "%";
|
| 121 |
$("thrv").innerHTML = `<span class="${e.threat >= .5 ? 'yes' : 'no'}">${e.threat >= .5 ? 'YES' : 'no'}</span> ${(100*e.threat).toFixed(0)}%`;
|
| 122 |
const tot = Object.values(e.total_actions).reduce((a, b) => a + b, 0);
|
|
|
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8">
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1">
|
| 6 |
+
<title>System01 plays Doom</title>
|
| 7 |
<style>
|
| 8 |
:root {
|
| 9 |
--bg: #0e0e0d; --panel: #171716; --line: #2a2a28; --ink: #f4f3ee; --ink2: #b7b5ab; --muted: #7d7b73;
|
|
|
|
| 22 |
.game { background: #000; border: 1px solid var(--line); border-radius: 10px; overflow: hidden; position: relative; }
|
| 23 |
.game img { width: 100%; display: block; image-rendering: pixelated; aspect-ratio: 4/3; }
|
| 24 |
.tag { position: absolute; left: 12px; top: 10px; background: rgba(0,0,0,.6); padding: 3px 8px; border-radius: 5px; font-size: 12px; color: var(--ink2); }
|
| 25 |
+
.row { display: grid; grid-template-columns: repeat(5, 1fr); gap: 10px; margin-top: 12px; }
|
| 26 |
.stat { background: var(--panel); border: 1px solid var(--line); border-radius: 8px; padding: 10px 12px; }
|
| 27 |
.stat .k { color: var(--muted); font-size: 11px; text-transform: uppercase; letter-spacing: .6px; }
|
| 28 |
.stat .v { font-size: 22px; font-weight: 650; font-variant-numeric: tabular-nums; }
|
|
|
|
| 51 |
</head>
|
| 52 |
<body>
|
| 53 |
<header>
|
| 54 |
+
<h1>System01 plays Doom</h1>
|
| 55 |
<span class="sub">no vision · no text generation · two typed questions answered in one forward pass per step</span>
|
| 56 |
<span class="sp"></span>
|
| 57 |
<span class="scen" id="scen"></span>
|
| 58 |
</header>
|
| 59 |
<main>
|
| 60 |
<section>
|
| 61 |
+
<div class="game"><img src="/stream.mjpg" alt="Live Doom game played by System01"><span class="tag" id="scenTag">—</span></div>
|
| 62 |
<div class="row">
|
| 63 |
+
<div class="stat"><div class="k">Key pressed</div><div class="v" id="key" style="font-size:16px">–</div></div>
|
| 64 |
<div class="stat"><div class="k">Latency / decision</div><div class="v" id="lat">–</div></div>
|
| 65 |
<div class="stat"><div class="k">Decisions / s</div><div class="v" id="dps">–</div></div>
|
| 66 |
<div class="stat"><div class="k">Health</div><div class="v" id="hp">–</div></div>
|
|
|
|
| 90 |
</div>
|
| 91 |
</aside>
|
| 92 |
</main>
|
| 93 |
+
<footer>Key pressed = ATTACK when P(crosshair on enemy) ≥ 0.5, otherwise the top choice · System01 · RTX 4090 · output tokens: 0 · no Doom data in training — the model reads the state text, the rules and the option names.</footer>
|
| 94 |
<script>
|
| 95 |
const ORDER = ["ATTACK","TURN_LEFT","TURN_RIGHT","MOVE_FORWARD","MOVE_LEFT","MOVE_RIGHT","MOVE_BACKWARD"];
|
| 96 |
const SCEN = ["defend_the_center","deadly_corridor","defend_the_line","basic"];
|
|
|
|
| 113 |
if (e.type === "episode") { episodes(e.episodes); return; }
|
| 114 |
if (e.scenario !== current) { current = e.scenario; scenButtons(); $("scenTag").textContent = current; }
|
| 115 |
$("lat").textContent = e.latency_ms.toFixed(1) + " ms";
|
| 116 |
+
$("key").textContent = e.action + (e.action === "ATTACK" && e.threat >= .5 ? " · noul fire" : "");
|
| 117 |
$("dps").textContent = e.dps.toFixed(0);
|
| 118 |
$("hp").textContent = Math.round(e.hp);
|
| 119 |
$("kills").textContent = Math.round(e.kills);
|
| 120 |
$("state").textContent = e.state;
|
| 121 |
+
const top = Object.keys(e.probs).reduce((a, b) => e.probs[a] >= e.probs[b] ? a : b);
|
| 122 |
+
bars($("acts"), e.probs, top, "act");
|
| 123 |
$("thr").style.width = (100 * e.threat).toFixed(1) + "%";
|
| 124 |
$("thrv").innerHTML = `<span class="${e.threat >= .5 ? 'yes' : 'no'}">${e.threat >= .5 ? 'YES' : 'no'}</span> ${(100*e.threat).toFixed(0)}%`;
|
| 125 |
const tot = Object.values(e.total_actions).reduce((a, b) => a + b, 0);
|
model.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 643835524
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1eea2b3f64a97914062a6d4e0f014ac44c06b738d3dc8edc5545b206ae13c65d
|
| 3 |
size 643835524
|
pyproject.toml
CHANGED
|
@@ -3,9 +3,9 @@ requires = ["setuptools>=61.0"]
|
|
| 3 |
build-backend = "setuptools.build_meta"
|
| 4 |
|
| 5 |
[project]
|
| 6 |
-
name = "
|
| 7 |
version = "1.0.0"
|
| 8 |
-
description = "
|
| 9 |
readme = "README.md"
|
| 10 |
requires-python = ">=3.10"
|
| 11 |
license = { text = "Apache-2.0" }
|
|
@@ -17,7 +17,7 @@ server = ["fastapi>=0.110", "uvicorn>=0.29"]
|
|
| 17 |
doom = ["vizdoom>=1.2", "pillow", "fastapi>=0.110", "uvicorn>=0.29"]
|
| 18 |
|
| 19 |
[tool.setuptools]
|
| 20 |
-
packages = ["
|
| 21 |
|
| 22 |
[project.urls]
|
| 23 |
-
Homepage = "https://huggingface.co/doofz/
|
|
|
|
| 3 |
build-backend = "setuptools.build_meta"
|
| 4 |
|
| 5 |
[project]
|
| 6 |
+
name = "system01"
|
| 7 |
version = "1.0.0"
|
| 8 |
+
description = "System01: calibrated, non-autoregressive typed decisions in one forward pass (100+ languages, Azerbaijani-first)"
|
| 9 |
readme = "README.md"
|
| 10 |
requires-python = ">=3.10"
|
| 11 |
license = { text = "Apache-2.0" }
|
|
|
|
| 17 |
doom = ["vizdoom>=1.2", "pillow", "fastapi>=0.110", "uvicorn>=0.29"]
|
| 18 |
|
| 19 |
[tool.setuptools]
|
| 20 |
+
packages = ["system01"]
|
| 21 |
|
| 22 |
[project.urls]
|
| 23 |
+
Homepage = "https://huggingface.co/doofz/System01"
|
results/az_categories.json
CHANGED
|
@@ -5,9 +5,9 @@
|
|
| 5 |
"ece": 0.3014,
|
| 6 |
"n": 1042
|
| 7 |
},
|
| 8 |
-
"
|
| 9 |
-
"acc": 0.
|
| 10 |
-
"ece": 0.
|
| 11 |
"n": 1042
|
| 12 |
}
|
| 13 |
},
|
|
@@ -17,9 +17,9 @@
|
|
| 17 |
"ece": 0.2625,
|
| 18 |
"n": 215
|
| 19 |
},
|
| 20 |
-
"
|
| 21 |
-
"acc": 0.
|
| 22 |
-
"ece": 0.
|
| 23 |
"n": 215
|
| 24 |
}
|
| 25 |
},
|
|
@@ -29,9 +29,9 @@
|
|
| 29 |
"ece": 0.2517,
|
| 30 |
"n": 235
|
| 31 |
},
|
| 32 |
-
"
|
| 33 |
-
"acc": 0.
|
| 34 |
-
"ece": 0.
|
| 35 |
"n": 235
|
| 36 |
}
|
| 37 |
},
|
|
@@ -41,9 +41,9 @@
|
|
| 41 |
"ece": 0.3393,
|
| 42 |
"n": 193
|
| 43 |
},
|
| 44 |
-
"
|
| 45 |
-
"acc": 0.
|
| 46 |
-
"ece": 0.
|
| 47 |
"n": 193
|
| 48 |
}
|
| 49 |
},
|
|
@@ -53,9 +53,9 @@
|
|
| 53 |
"ece": 0.2672,
|
| 54 |
"n": 181
|
| 55 |
},
|
| 56 |
-
"
|
| 57 |
"acc": 0.9558,
|
| 58 |
-
"ece": 0.
|
| 59 |
"n": 181
|
| 60 |
}
|
| 61 |
},
|
|
@@ -65,9 +65,9 @@
|
|
| 65 |
"ece": 0.439,
|
| 66 |
"n": 431
|
| 67 |
},
|
| 68 |
-
"
|
| 69 |
-
"acc": 0.
|
| 70 |
-
"ece": 0.
|
| 71 |
"n": 431
|
| 72 |
}
|
| 73 |
},
|
|
@@ -77,9 +77,9 @@
|
|
| 77 |
"ece": 0.2145,
|
| 78 |
"n": 269
|
| 79 |
},
|
| 80 |
-
"
|
| 81 |
-
"acc": 0.
|
| 82 |
-
"ece": 0.
|
| 83 |
"n": 269
|
| 84 |
}
|
| 85 |
},
|
|
@@ -89,9 +89,9 @@
|
|
| 89 |
"ece": 0.365,
|
| 90 |
"n": 1000
|
| 91 |
},
|
| 92 |
-
"
|
| 93 |
-
"acc": 0.
|
| 94 |
-
"ece": 0.
|
| 95 |
"n": 1000
|
| 96 |
}
|
| 97 |
},
|
|
@@ -101,9 +101,9 @@
|
|
| 101 |
"ece": 0.2101,
|
| 102 |
"n": 1000
|
| 103 |
},
|
| 104 |
-
"
|
| 105 |
-
"acc": 0.
|
| 106 |
-
"ece": 0.
|
| 107 |
"n": 1000
|
| 108 |
}
|
| 109 |
},
|
|
@@ -113,9 +113,9 @@
|
|
| 113 |
"ece": 0.1768,
|
| 114 |
"n": 1000
|
| 115 |
},
|
| 116 |
-
"
|
| 117 |
-
"acc": 0.
|
| 118 |
-
"ece": 0.
|
| 119 |
"n": 1000
|
| 120 |
}
|
| 121 |
},
|
|
@@ -125,9 +125,9 @@
|
|
| 125 |
"ece": 0.2098,
|
| 126 |
"n": 204
|
| 127 |
},
|
| 128 |
-
"
|
| 129 |
-
"acc": 0.
|
| 130 |
-
"ece": 0.
|
| 131 |
"n": 204
|
| 132 |
}
|
| 133 |
},
|
|
@@ -137,9 +137,9 @@
|
|
| 137 |
"ece": 0.5769,
|
| 138 |
"n": 32
|
| 139 |
},
|
| 140 |
-
"
|
| 141 |
-
"acc": 0.
|
| 142 |
-
"ece": 0.
|
| 143 |
"n": 32
|
| 144 |
}
|
| 145 |
}
|
|
|
|
| 5 |
"ece": 0.3014,
|
| 6 |
"n": 1042
|
| 7 |
},
|
| 8 |
+
"system01": {
|
| 9 |
+
"acc": 0.9511,
|
| 10 |
+
"ece": 0.0129,
|
| 11 |
"n": 1042
|
| 12 |
}
|
| 13 |
},
|
|
|
|
| 17 |
"ece": 0.2625,
|
| 18 |
"n": 215
|
| 19 |
},
|
| 20 |
+
"system01": {
|
| 21 |
+
"acc": 0.9023,
|
| 22 |
+
"ece": 0.0492,
|
| 23 |
"n": 215
|
| 24 |
}
|
| 25 |
},
|
|
|
|
| 29 |
"ece": 0.2517,
|
| 30 |
"n": 235
|
| 31 |
},
|
| 32 |
+
"system01": {
|
| 33 |
+
"acc": 0.9319,
|
| 34 |
+
"ece": 0.034,
|
| 35 |
"n": 235
|
| 36 |
}
|
| 37 |
},
|
|
|
|
| 41 |
"ece": 0.3393,
|
| 42 |
"n": 193
|
| 43 |
},
|
| 44 |
+
"system01": {
|
| 45 |
+
"acc": 0.7358,
|
| 46 |
+
"ece": 0.0654,
|
| 47 |
"n": 193
|
| 48 |
}
|
| 49 |
},
|
|
|
|
| 53 |
"ece": 0.2672,
|
| 54 |
"n": 181
|
| 55 |
},
|
| 56 |
+
"system01": {
|
| 57 |
"acc": 0.9558,
|
| 58 |
+
"ece": 0.0364,
|
| 59 |
"n": 181
|
| 60 |
}
|
| 61 |
},
|
|
|
|
| 65 |
"ece": 0.439,
|
| 66 |
"n": 431
|
| 67 |
},
|
| 68 |
+
"system01": {
|
| 69 |
+
"acc": 0.8886,
|
| 70 |
+
"ece": 0.0423,
|
| 71 |
"n": 431
|
| 72 |
}
|
| 73 |
},
|
|
|
|
| 77 |
"ece": 0.2145,
|
| 78 |
"n": 269
|
| 79 |
},
|
| 80 |
+
"system01": {
|
| 81 |
+
"acc": 0.9219,
|
| 82 |
+
"ece": 0.0436,
|
| 83 |
"n": 269
|
| 84 |
}
|
| 85 |
},
|
|
|
|
| 89 |
"ece": 0.365,
|
| 90 |
"n": 1000
|
| 91 |
},
|
| 92 |
+
"system01": {
|
| 93 |
+
"acc": 0.818,
|
| 94 |
+
"ece": 0.0311,
|
| 95 |
"n": 1000
|
| 96 |
}
|
| 97 |
},
|
|
|
|
| 101 |
"ece": 0.2101,
|
| 102 |
"n": 1000
|
| 103 |
},
|
| 104 |
+
"system01": {
|
| 105 |
+
"acc": 0.885,
|
| 106 |
+
"ece": 0.0442,
|
| 107 |
"n": 1000
|
| 108 |
}
|
| 109 |
},
|
|
|
|
| 113 |
"ece": 0.1768,
|
| 114 |
"n": 1000
|
| 115 |
},
|
| 116 |
+
"system01": {
|
| 117 |
+
"acc": 0.849,
|
| 118 |
+
"ece": 0.0907,
|
| 119 |
"n": 1000
|
| 120 |
}
|
| 121 |
},
|
|
|
|
| 125 |
"ece": 0.2098,
|
| 126 |
"n": 204
|
| 127 |
},
|
| 128 |
+
"system01": {
|
| 129 |
+
"acc": 0.7402,
|
| 130 |
+
"ece": 0.1377,
|
| 131 |
"n": 204
|
| 132 |
}
|
| 133 |
},
|
|
|
|
| 137 |
"ece": 0.5769,
|
| 138 |
"n": 32
|
| 139 |
},
|
| 140 |
+
"system01": {
|
| 141 |
+
"acc": 0.9375,
|
| 142 |
+
"ece": 0.077,
|
| 143 |
"n": 32
|
| 144 |
}
|
| 145 |
}
|
results/doom.json
CHANGED
|
@@ -1,17 +1,17 @@
|
|
| 1 |
{
|
| 2 |
"defend_the_center": {
|
| 3 |
-
"
|
| 4 |
-
"kills_mean": 13.
|
| 5 |
-
"kills_std": 1.
|
| 6 |
-
"reward_mean": 12.
|
| 7 |
-
"steps_mean":
|
| 8 |
"episodes": 20,
|
| 9 |
"actions": {
|
| 10 |
-
"ATTACK":
|
| 11 |
-
"TURN_LEFT":
|
| 12 |
-
"TURN_RIGHT":
|
| 13 |
},
|
| 14 |
-
"latency_p50_ms": 10.
|
| 15 |
},
|
| 16 |
"random": {
|
| 17 |
"kills_mean": 1.45,
|
|
@@ -27,18 +27,18 @@
|
|
| 27 |
}
|
| 28 |
},
|
| 29 |
"defend_the_line": {
|
| 30 |
-
"
|
| 31 |
-
"kills_mean":
|
| 32 |
-
"kills_std":
|
| 33 |
-
"reward_mean":
|
| 34 |
-
"steps_mean":
|
| 35 |
"episodes": 20,
|
| 36 |
"actions": {
|
| 37 |
-
"ATTACK":
|
| 38 |
-
"TURN_RIGHT":
|
| 39 |
-
"TURN_LEFT":
|
| 40 |
},
|
| 41 |
-
"latency_p50_ms":
|
| 42 |
},
|
| 43 |
"random": {
|
| 44 |
"kills_mean": 7.65,
|
|
@@ -54,7 +54,7 @@
|
|
| 54 |
}
|
| 55 |
},
|
| 56 |
"basic": {
|
| 57 |
-
"
|
| 58 |
"kills_mean": 1.0,
|
| 59 |
"kills_std": 0.0,
|
| 60 |
"reward_mean": 79.25,
|
|
@@ -65,7 +65,7 @@
|
|
| 65 |
"ATTACK": 43,
|
| 66 |
"MOVE_LEFT": 56
|
| 67 |
},
|
| 68 |
-
"latency_p50_ms": 10.
|
| 69 |
},
|
| 70 |
"random": {
|
| 71 |
"kills_mean": 0.8,
|
|
@@ -81,21 +81,21 @@
|
|
| 81 |
}
|
| 82 |
},
|
| 83 |
"deadly_corridor": {
|
| 84 |
-
"
|
| 85 |
-
"kills_mean":
|
| 86 |
-
"kills_std":
|
| 87 |
-
"reward_mean":
|
| 88 |
-
"steps_mean":
|
| 89 |
"episodes": 20,
|
| 90 |
"actions": {
|
| 91 |
-
"TURN_RIGHT":
|
| 92 |
-
"ATTACK":
|
| 93 |
-
"MOVE_BACKWARD":
|
| 94 |
-
"TURN_LEFT":
|
| 95 |
-
"MOVE_FORWARD":
|
| 96 |
-
"MOVE_RIGHT":
|
| 97 |
},
|
| 98 |
-
"latency_p50_ms": 10.
|
| 99 |
},
|
| 100 |
"random": {
|
| 101 |
"kills_mean": 0.8,
|
|
|
|
| 1 |
{
|
| 2 |
"defend_the_center": {
|
| 3 |
+
"system01": {
|
| 4 |
+
"kills_mean": 13.55,
|
| 5 |
+
"kills_std": 1.883480820183736,
|
| 6 |
+
"reward_mean": 12.55,
|
| 7 |
+
"steps_mean": 208.4,
|
| 8 |
"episodes": 20,
|
| 9 |
"actions": {
|
| 10 |
+
"ATTACK": 2289,
|
| 11 |
+
"TURN_LEFT": 1500,
|
| 12 |
+
"TURN_RIGHT": 379
|
| 13 |
},
|
| 14 |
+
"latency_p50_ms": 10.005719494074583
|
| 15 |
},
|
| 16 |
"random": {
|
| 17 |
"kills_mean": 1.45,
|
|
|
|
| 27 |
}
|
| 28 |
},
|
| 29 |
"defend_the_line": {
|
| 30 |
+
"system01": {
|
| 31 |
+
"kills_mean": 10.65,
|
| 32 |
+
"kills_std": 2.650943228362313,
|
| 33 |
+
"reward_mean": 9.65,
|
| 34 |
+
"steps_mean": 138.35,
|
| 35 |
"episodes": 20,
|
| 36 |
"actions": {
|
| 37 |
+
"ATTACK": 572,
|
| 38 |
+
"TURN_RIGHT": 341,
|
| 39 |
+
"TURN_LEFT": 1854
|
| 40 |
},
|
| 41 |
+
"latency_p50_ms": 9.979521972127259
|
| 42 |
},
|
| 43 |
"random": {
|
| 44 |
"kills_mean": 7.65,
|
|
|
|
| 54 |
}
|
| 55 |
},
|
| 56 |
"basic": {
|
| 57 |
+
"system01": {
|
| 58 |
"kills_mean": 1.0,
|
| 59 |
"kills_std": 0.0,
|
| 60 |
"reward_mean": 79.25,
|
|
|
|
| 65 |
"ATTACK": 43,
|
| 66 |
"MOVE_LEFT": 56
|
| 67 |
},
|
| 68 |
+
"latency_p50_ms": 10.276200016960502
|
| 69 |
},
|
| 70 |
"random": {
|
| 71 |
"kills_mean": 0.8,
|
|
|
|
| 81 |
}
|
| 82 |
},
|
| 83 |
"deadly_corridor": {
|
| 84 |
+
"system01": {
|
| 85 |
+
"kills_mean": 4.45,
|
| 86 |
+
"kills_std": 1.116915395184434,
|
| 87 |
+
"reward_mean": 698.8700164794922,
|
| 88 |
+
"steps_mean": 759.05,
|
| 89 |
"episodes": 20,
|
| 90 |
"actions": {
|
| 91 |
+
"TURN_RIGHT": 175,
|
| 92 |
+
"ATTACK": 11092,
|
| 93 |
+
"MOVE_BACKWARD": 1405,
|
| 94 |
+
"TURN_LEFT": 148,
|
| 95 |
+
"MOVE_FORWARD": 1850,
|
| 96 |
+
"MOVE_RIGHT": 511
|
| 97 |
},
|
| 98 |
+
"latency_p50_ms": 10.548294987529516
|
| 99 |
},
|
| 100 |
"random": {
|
| 101 |
"kills_mean": 0.8,
|
results/final_release.json
CHANGED
|
@@ -9,13 +9,21 @@
|
|
| 9 |
"n": 42,
|
| 10 |
"sec": 0.21
|
| 11 |
},
|
| 12 |
-
"
|
| 13 |
"acc": 0.9048,
|
| 14 |
"ece": 0.0925,
|
| 15 |
"conf": 0.9033,
|
| 16 |
"brier": 0.1018,
|
| 17 |
"n": 42,
|
| 18 |
"sec": 0.04
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
}
|
| 20 |
},
|
| 21 |
"Synthetic routing (held-out schemas)": {
|
|
@@ -25,15 +33,23 @@
|
|
| 25 |
"conf": 0.6079,
|
| 26 |
"brier": 0.8977,
|
| 27 |
"n": 1500,
|
| 28 |
-
"sec": 1.
|
| 29 |
},
|
| 30 |
-
"
|
| 31 |
"acc": 0.952,
|
| 32 |
"ece": 0.0151,
|
| 33 |
"conf": 0.9634,
|
| 34 |
"brier": 0.0749,
|
| 35 |
"n": 1500,
|
| 36 |
-
"sec": 1.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
}
|
| 38 |
},
|
| 39 |
"Synthetic decisions (held-out schemas)": {
|
|
@@ -43,15 +59,23 @@
|
|
| 43 |
"conf": 0.7683,
|
| 44 |
"brier": 0.5892,
|
| 45 |
"n": 1867,
|
| 46 |
-
"sec": 1.
|
| 47 |
},
|
| 48 |
-
"
|
| 49 |
"acc": 0.8597,
|
| 50 |
"ece": 0.0241,
|
| 51 |
"conf": 0.8831,
|
| 52 |
"brier": 0.1724,
|
| 53 |
"n": 1867,
|
| 54 |
"sec": 1.35
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
}
|
| 56 |
},
|
| 57 |
"Synthetic guard (held-out schemas)": {
|
|
@@ -63,13 +87,21 @@
|
|
| 63 |
"n": 435,
|
| 64 |
"sec": 0.36
|
| 65 |
},
|
| 66 |
-
"
|
| 67 |
"acc": 0.9678,
|
| 68 |
"ece": 0.0287,
|
| 69 |
"conf": 0.9529,
|
| 70 |
"brier": 0.031,
|
| 71 |
"n": 435,
|
| 72 |
"sec": 0.21
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 73 |
}
|
| 74 |
},
|
| 75 |
"Synthetic taxonomy (held-out schemas)": {
|
|
@@ -79,15 +111,23 @@
|
|
| 79 |
"conf": 0.7117,
|
| 80 |
"brier": 0.957,
|
| 81 |
"n": 1132,
|
| 82 |
-
"sec": 6.
|
| 83 |
},
|
| 84 |
-
"
|
| 85 |
"acc": 0.8719,
|
| 86 |
"ece": 0.0607,
|
| 87 |
"conf": 0.925,
|
| 88 |
"brier": 0.1943,
|
| 89 |
"n": 1132,
|
| 90 |
-
"sec": 6.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 91 |
}
|
| 92 |
},
|
| 93 |
"Synthetic emotion (held-out schemas)": {
|
|
@@ -99,13 +139,21 @@
|
|
| 99 |
"n": 505,
|
| 100 |
"sec": 0.31
|
| 101 |
},
|
| 102 |
-
"
|
| 103 |
"acc": 0.9505,
|
| 104 |
"ece": 0.0315,
|
| 105 |
"conf": 0.9249,
|
| 106 |
"brier": 0.0799,
|
| 107 |
"n": 505,
|
| 108 |
"sec": 0.31
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
}
|
| 110 |
},
|
| 111 |
"AG News (4)": {
|
|
@@ -115,14 +163,22 @@
|
|
| 115 |
"conf": 0.9674,
|
| 116 |
"brier": 0.1015,
|
| 117 |
"n": 1000,
|
| 118 |
-
"sec": 0.
|
| 119 |
},
|
| 120 |
-
"
|
| 121 |
"acc": 0.931,
|
| 122 |
"ece": 0.0247,
|
| 123 |
"conf": 0.9438,
|
| 124 |
"brier": 0.114,
|
| 125 |
"n": 1000,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 126 |
"sec": 0.66
|
| 127 |
}
|
| 128 |
},
|
|
@@ -135,13 +191,21 @@
|
|
| 135 |
"n": 1025,
|
| 136 |
"sec": 0.75
|
| 137 |
},
|
| 138 |
-
"
|
| 139 |
"acc": 0.4537,
|
| 140 |
"ece": 0.2924,
|
| 141 |
"conf": 0.746,
|
| 142 |
"brier": 0.7854,
|
| 143 |
"n": 1025,
|
| 144 |
"sec": 0.75
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 145 |
}
|
| 146 |
},
|
| 147 |
"Banking77 (77)": {
|
|
@@ -151,15 +215,23 @@
|
|
| 151 |
"conf": 0.8734,
|
| 152 |
"brier": 0.8089,
|
| 153 |
"n": 1001,
|
| 154 |
-
"sec": 8.
|
| 155 |
},
|
| 156 |
-
"
|
| 157 |
"acc": 0.6234,
|
| 158 |
"ece": 0.1326,
|
| 159 |
"conf": 0.756,
|
| 160 |
"brier": 0.5411,
|
| 161 |
"n": 1001,
|
| 162 |
-
"sec": 8.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 163 |
}
|
| 164 |
},
|
| 165 |
"SMS spam (yes/no)": {
|
|
@@ -169,15 +241,23 @@
|
|
| 169 |
"conf": 0.9435,
|
| 170 |
"brier": 0.4624,
|
| 171 |
"n": 1000,
|
| 172 |
-
"sec": 0.
|
| 173 |
},
|
| 174 |
-
"
|
| 175 |
"acc": 0.936,
|
| 176 |
"ece": 0.0387,
|
| 177 |
"conf": 0.9057,
|
| 178 |
"brier": 0.095,
|
| 179 |
"n": 1000,
|
| 180 |
-
"sec": 0.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 181 |
}
|
| 182 |
},
|
| 183 |
"AZ app-review sentiment": {
|
|
@@ -189,13 +269,21 @@
|
|
| 189 |
"n": 1000,
|
| 190 |
"sec": 0.37
|
| 191 |
},
|
| 192 |
-
"
|
| 193 |
"acc": 0.857,
|
| 194 |
"ece": 0.1112,
|
| 195 |
"conf": 0.7462,
|
| 196 |
"brier": 0.2424,
|
| 197 |
"n": 1000,
|
| 198 |
"sec": 0.53
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 199 |
}
|
| 200 |
},
|
| 201 |
"MASSIVE-intent@20 (az)": {
|
|
@@ -205,15 +293,23 @@
|
|
| 205 |
"conf": 0.6292,
|
| 206 |
"brier": 0.8902,
|
| 207 |
"n": 1000,
|
| 208 |
-
"sec": 1.
|
| 209 |
},
|
| 210 |
-
"
|
| 211 |
"acc": 0.898,
|
| 212 |
"ece": 0.0459,
|
| 213 |
"conf": 0.858,
|
| 214 |
"brier": 0.1482,
|
| 215 |
"n": 1000,
|
| 216 |
-
"sec": 1.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 217 |
}
|
| 218 |
},
|
| 219 |
"Intent yes/no (az)": {
|
|
@@ -225,13 +321,21 @@
|
|
| 225 |
"n": 1000,
|
| 226 |
"sec": 0.35
|
| 227 |
},
|
| 228 |
-
"
|
| 229 |
"acc": 0.944,
|
| 230 |
"ece": 0.0161,
|
| 231 |
"conf": 0.9465,
|
| 232 |
"brier": 0.0389,
|
| 233 |
"n": 1000,
|
| 234 |
-
"sec": 0.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 235 |
}
|
| 236 |
},
|
| 237 |
"MASSIVE-intent@20 (en)": {
|
|
@@ -241,15 +345,23 @@
|
|
| 241 |
"conf": 0.7832,
|
| 242 |
"brier": 0.5463,
|
| 243 |
"n": 1000,
|
| 244 |
-
"sec": 1.
|
| 245 |
},
|
| 246 |
-
"
|
| 247 |
"acc": 0.932,
|
| 248 |
"ece": 0.0563,
|
| 249 |
"conf": 0.8768,
|
| 250 |
"brier": 0.1142,
|
| 251 |
"n": 1000,
|
| 252 |
-
"sec": 1.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 253 |
}
|
| 254 |
},
|
| 255 |
"Intent yes/no (en)": {
|
|
@@ -261,13 +373,21 @@
|
|
| 261 |
"n": 1000,
|
| 262 |
"sec": 0.47
|
| 263 |
},
|
| 264 |
-
"
|
| 265 |
"acc": 0.956,
|
| 266 |
"ece": 0.0183,
|
| 267 |
"conf": 0.9431,
|
| 268 |
"brier": 0.0324,
|
| 269 |
"n": 1000,
|
| 270 |
"sec": 0.32
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 271 |
}
|
| 272 |
},
|
| 273 |
"MASSIVE-scenario (en)": {
|
|
@@ -277,15 +397,23 @@
|
|
| 277 |
"conf": 0.777,
|
| 278 |
"brier": 0.4383,
|
| 279 |
"n": 500,
|
| 280 |
-
"sec": 0.
|
| 281 |
},
|
| 282 |
-
"
|
| 283 |
"acc": 0.91,
|
| 284 |
"ece": 0.0651,
|
| 285 |
"conf": 0.8491,
|
| 286 |
"brier": 0.1413,
|
| 287 |
"n": 500,
|
| 288 |
"sec": 0.74
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 289 |
}
|
| 290 |
},
|
| 291 |
"MASSIVE-scenario (az)": {
|
|
@@ -295,14 +423,22 @@
|
|
| 295 |
"conf": 0.6237,
|
| 296 |
"brier": 0.7993,
|
| 297 |
"n": 500,
|
| 298 |
-
"sec": 0.
|
| 299 |
},
|
| 300 |
-
"
|
| 301 |
"acc": 0.884,
|
| 302 |
"ece": 0.0527,
|
| 303 |
"conf": 0.8509,
|
| 304 |
"brier": 0.1745,
|
| 305 |
"n": 500,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 306 |
"sec": 0.77
|
| 307 |
}
|
| 308 |
},
|
|
@@ -313,15 +449,23 @@
|
|
| 313 |
"conf": 0.734,
|
| 314 |
"brier": 0.6105,
|
| 315 |
"n": 500,
|
| 316 |
-
"sec": 0.
|
| 317 |
},
|
| 318 |
-
"
|
| 319 |
"acc": 0.886,
|
| 320 |
"ece": 0.059,
|
| 321 |
"conf": 0.8326,
|
| 322 |
"brier": 0.1712,
|
| 323 |
"n": 500,
|
| 324 |
-
"sec": 0.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 325 |
}
|
| 326 |
},
|
| 327 |
"MASSIVE-scenario (tr)": {
|
|
@@ -331,14 +475,22 @@
|
|
| 331 |
"conf": 0.6785,
|
| 332 |
"brier": 0.7032,
|
| 333 |
"n": 500,
|
| 334 |
-
"sec": 0.
|
| 335 |
},
|
| 336 |
-
"
|
| 337 |
"acc": 0.886,
|
| 338 |
"ece": 0.0729,
|
| 339 |
"conf": 0.8287,
|
| 340 |
"brier": 0.1687,
|
| 341 |
"n": 500,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 342 |
"sec": 0.75
|
| 343 |
}
|
| 344 |
},
|
|
@@ -349,14 +501,22 @@
|
|
| 349 |
"conf": 0.7256,
|
| 350 |
"brier": 0.6269,
|
| 351 |
"n": 500,
|
| 352 |
-
"sec": 0.
|
| 353 |
},
|
| 354 |
-
"
|
| 355 |
"acc": 0.884,
|
| 356 |
"ece": 0.057,
|
| 357 |
"conf": 0.8343,
|
| 358 |
"brier": 0.1739,
|
| 359 |
"n": 500,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 360 |
"sec": 0.75
|
| 361 |
}
|
| 362 |
},
|
|
@@ -367,15 +527,23 @@
|
|
| 367 |
"conf": 0.738,
|
| 368 |
"brier": 0.5843,
|
| 369 |
"n": 500,
|
| 370 |
-
"sec": 0.
|
| 371 |
},
|
| 372 |
-
"
|
| 373 |
"acc": 0.894,
|
| 374 |
"ece": 0.0566,
|
| 375 |
"conf": 0.8392,
|
| 376 |
"brier": 0.1544,
|
| 377 |
"n": 500,
|
| 378 |
"sec": 0.75
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 379 |
}
|
| 380 |
},
|
| 381 |
"MASSIVE-scenario (es)": {
|
|
@@ -385,15 +553,23 @@
|
|
| 385 |
"conf": 0.7339,
|
| 386 |
"brier": 0.6289,
|
| 387 |
"n": 500,
|
| 388 |
-
"sec": 0.
|
| 389 |
},
|
| 390 |
-
"
|
| 391 |
"acc": 0.878,
|
| 392 |
"ece": 0.0574,
|
| 393 |
"conf": 0.8233,
|
| 394 |
"brier": 0.1851,
|
| 395 |
"n": 500,
|
| 396 |
-
"sec": 0.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 397 |
}
|
| 398 |
},
|
| 399 |
"MASSIVE-scenario (ar)": {
|
|
@@ -403,15 +579,23 @@
|
|
| 403 |
"conf": 0.6603,
|
| 404 |
"brier": 0.7943,
|
| 405 |
"n": 500,
|
| 406 |
-
"sec": 0.
|
| 407 |
},
|
| 408 |
-
"
|
| 409 |
"acc": 0.828,
|
| 410 |
"ece": 0.0618,
|
| 411 |
"conf": 0.8009,
|
| 412 |
"brier": 0.2499,
|
| 413 |
"n": 500,
|
| 414 |
-
"sec": 0.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 415 |
}
|
| 416 |
},
|
| 417 |
"MASSIVE-scenario (hi)": {
|
|
@@ -423,13 +607,21 @@
|
|
| 423 |
"n": 500,
|
| 424 |
"sec": 0.76
|
| 425 |
},
|
| 426 |
-
"
|
| 427 |
"acc": 0.86,
|
| 428 |
"ece": 0.0466,
|
| 429 |
"conf": 0.8197,
|
| 430 |
"brier": 0.2053,
|
| 431 |
"n": 500,
|
| 432 |
"sec": 0.77
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 433 |
}
|
| 434 |
},
|
| 435 |
"MASSIVE-scenario (zh-CN)": {
|
|
@@ -441,13 +633,21 @@
|
|
| 441 |
"n": 500,
|
| 442 |
"sec": 0.74
|
| 443 |
},
|
| 444 |
-
"
|
| 445 |
"acc": 0.872,
|
| 446 |
"ece": 0.0647,
|
| 447 |
"conf": 0.8212,
|
| 448 |
"brier": 0.1766,
|
| 449 |
"n": 500,
|
| 450 |
"sec": 0.74
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 451 |
}
|
| 452 |
},
|
| 453 |
"MASSIVE-scenario (fa)": {
|
|
@@ -457,14 +657,22 @@
|
|
| 457 |
"conf": 0.6965,
|
| 458 |
"brier": 0.7663,
|
| 459 |
"n": 500,
|
| 460 |
-
"sec": 0.
|
| 461 |
},
|
| 462 |
-
"
|
| 463 |
"acc": 0.876,
|
| 464 |
"ece": 0.0617,
|
| 465 |
"conf": 0.8308,
|
| 466 |
"brier": 0.1695,
|
| 467 |
"n": 500,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 468 |
"sec": 0.76
|
| 469 |
}
|
| 470 |
},
|
|
@@ -475,14 +683,22 @@
|
|
| 475 |
"conf": 0.4054,
|
| 476 |
"brier": 1.036,
|
| 477 |
"n": 500,
|
| 478 |
-
"sec": 0.
|
| 479 |
},
|
| 480 |
-
"
|
| 481 |
"acc": 0.79,
|
| 482 |
"ece": 0.0572,
|
| 483 |
"conf": 0.7382,
|
| 484 |
"brier": 0.2991,
|
| 485 |
"n": 500,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 486 |
"sec": 0.8
|
| 487 |
}
|
| 488 |
},
|
|
@@ -493,15 +709,23 @@
|
|
| 493 |
"conf": 0.7711,
|
| 494 |
"brier": 0.6096,
|
| 495 |
"n": 500,
|
| 496 |
-
"sec": 0.
|
| 497 |
},
|
| 498 |
-
"
|
| 499 |
"acc": 0.898,
|
| 500 |
"ece": 0.0546,
|
| 501 |
"conf": 0.8542,
|
| 502 |
"brier": 0.1455,
|
| 503 |
"n": 500,
|
| 504 |
-
"sec": 0.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 505 |
}
|
| 506 |
},
|
| 507 |
"MASSIVE-scenario (ko)": {
|
|
@@ -511,15 +735,23 @@
|
|
| 511 |
"conf": 0.7026,
|
| 512 |
"brier": 0.7354,
|
| 513 |
"n": 500,
|
| 514 |
-
"sec": 0.
|
| 515 |
},
|
| 516 |
-
"
|
| 517 |
"acc": 0.878,
|
| 518 |
"ece": 0.0532,
|
| 519 |
"conf": 0.8363,
|
| 520 |
"brier": 0.1827,
|
| 521 |
"n": 500,
|
| 522 |
-
"sec": 0.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 523 |
}
|
| 524 |
},
|
| 525 |
"SIB-200 (az)": {
|
|
@@ -529,14 +761,22 @@
|
|
| 529 |
"conf": 0.858,
|
| 530 |
"brier": 0.5221,
|
| 531 |
"n": 204,
|
| 532 |
-
"sec": 0.
|
| 533 |
},
|
| 534 |
-
"
|
| 535 |
"acc": 0.7451,
|
| 536 |
"ece": 0.1092,
|
| 537 |
"conf": 0.8319,
|
| 538 |
"brier": 0.3752,
|
| 539 |
"n": 204,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 540 |
"sec": 0.18
|
| 541 |
}
|
| 542 |
},
|
|
@@ -549,13 +789,21 @@
|
|
| 549 |
"n": 204,
|
| 550 |
"sec": 0.15
|
| 551 |
},
|
| 552 |
-
"
|
| 553 |
"acc": 0.8529,
|
| 554 |
"ece": 0.0799,
|
| 555 |
"conf": 0.802,
|
| 556 |
"brier": 0.261,
|
| 557 |
"n": 204,
|
| 558 |
"sec": 0.15
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 559 |
}
|
| 560 |
},
|
| 561 |
"SIB-200 (ru)": {
|
|
@@ -567,13 +815,21 @@
|
|
| 567 |
"n": 204,
|
| 568 |
"sec": 0.16
|
| 569 |
},
|
| 570 |
-
"
|
| 571 |
"acc": 0.7598,
|
| 572 |
"ece": 0.0736,
|
| 573 |
"conf": 0.8153,
|
| 574 |
"brier": 0.3231,
|
| 575 |
"n": 204,
|
| 576 |
"sec": 0.17
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 577 |
}
|
| 578 |
},
|
| 579 |
"SIB-200 (tr)": {
|
|
@@ -583,15 +839,23 @@
|
|
| 583 |
"conf": 0.8547,
|
| 584 |
"brier": 0.4185,
|
| 585 |
"n": 204,
|
| 586 |
-
"sec": 0.
|
| 587 |
},
|
| 588 |
-
"
|
| 589 |
"acc": 0.7696,
|
| 590 |
"ece": 0.1237,
|
| 591 |
"conf": 0.8294,
|
| 592 |
"brier": 0.3642,
|
| 593 |
"n": 204,
|
| 594 |
"sec": 0.16
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 595 |
}
|
| 596 |
},
|
| 597 |
"SIB-200 (de)": {
|
|
@@ -603,13 +867,21 @@
|
|
| 603 |
"n": 204,
|
| 604 |
"sec": 0.16
|
| 605 |
},
|
| 606 |
-
"
|
| 607 |
"acc": 0.8039,
|
| 608 |
"ece": 0.0818,
|
| 609 |
"conf": 0.8238,
|
| 610 |
"brier": 0.293,
|
| 611 |
"n": 204,
|
| 612 |
"sec": 0.16
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 613 |
}
|
| 614 |
},
|
| 615 |
"SIB-200 (ar)": {
|
|
@@ -619,15 +891,23 @@
|
|
| 619 |
"conf": 0.8589,
|
| 620 |
"brier": 0.4077,
|
| 621 |
"n": 204,
|
| 622 |
-
"sec": 0.
|
| 623 |
},
|
| 624 |
-
"
|
| 625 |
"acc": 0.7745,
|
| 626 |
"ece": 0.0704,
|
| 627 |
"conf": 0.8245,
|
| 628 |
"brier": 0.333,
|
| 629 |
"n": 204,
|
| 630 |
"sec": 0.17
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 631 |
}
|
| 632 |
},
|
| 633 |
"SIB-200 (hi)": {
|
|
@@ -637,15 +917,23 @@
|
|
| 637 |
"conf": 0.8433,
|
| 638 |
"brier": 0.4835,
|
| 639 |
"n": 204,
|
| 640 |
-
"sec": 0.
|
| 641 |
},
|
| 642 |
-
"
|
| 643 |
"acc": 0.7157,
|
| 644 |
"ece": 0.0998,
|
| 645 |
"conf": 0.8112,
|
| 646 |
"brier": 0.4172,
|
| 647 |
"n": 204,
|
| 648 |
"sec": 0.18
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 649 |
}
|
| 650 |
},
|
| 651 |
"SIB-200 (zh)": {
|
|
@@ -657,13 +945,21 @@
|
|
| 657 |
"n": 204,
|
| 658 |
"sec": 0.16
|
| 659 |
},
|
| 660 |
-
"
|
| 661 |
"acc": 0.8039,
|
| 662 |
"ece": 0.0891,
|
| 663 |
"conf": 0.818,
|
| 664 |
"brier": 0.3114,
|
| 665 |
"n": 204,
|
| 666 |
-
"sec": 0.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 667 |
}
|
| 668 |
},
|
| 669 |
"SIB-200 (kk)": {
|
|
@@ -673,14 +969,22 @@
|
|
| 673 |
"conf": 0.815,
|
| 674 |
"brier": 0.4644,
|
| 675 |
"n": 204,
|
| 676 |
-
"sec": 0.
|
| 677 |
},
|
| 678 |
-
"
|
| 679 |
"acc": 0.7402,
|
| 680 |
"ece": 0.0659,
|
| 681 |
"conf": 0.7902,
|
| 682 |
"brier": 0.3776,
|
| 683 |
"n": 204,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 684 |
"sec": 0.19
|
| 685 |
}
|
| 686 |
},
|
|
@@ -691,15 +995,23 @@
|
|
| 691 |
"conf": 0.8053,
|
| 692 |
"brier": 0.6262,
|
| 693 |
"n": 204,
|
| 694 |
-
"sec": 0.
|
| 695 |
},
|
| 696 |
-
"
|
| 697 |
"acc": 0.6667,
|
| 698 |
"ece": 0.1444,
|
| 699 |
"conf": 0.7902,
|
| 700 |
"brier": 0.4792,
|
| 701 |
"n": 204,
|
| 702 |
"sec": 0.18
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 703 |
}
|
| 704 |
}
|
| 705 |
},
|
|
|
|
| 9 |
"n": 42,
|
| 10 |
"sec": 0.21
|
| 11 |
},
|
| 12 |
+
"previous": {
|
| 13 |
"acc": 0.9048,
|
| 14 |
"ece": 0.0925,
|
| 15 |
"conf": 0.9033,
|
| 16 |
"brier": 0.1018,
|
| 17 |
"n": 42,
|
| 18 |
"sec": 0.04
|
| 19 |
+
},
|
| 20 |
+
"system01": {
|
| 21 |
+
"acc": 0.9286,
|
| 22 |
+
"ece": 0.0817,
|
| 23 |
+
"conf": 0.8935,
|
| 24 |
+
"brier": 0.0833,
|
| 25 |
+
"n": 42,
|
| 26 |
+
"sec": 0.04
|
| 27 |
}
|
| 28 |
},
|
| 29 |
"Synthetic routing (held-out schemas)": {
|
|
|
|
| 33 |
"conf": 0.6079,
|
| 34 |
"brier": 0.8977,
|
| 35 |
"n": 1500,
|
| 36 |
+
"sec": 1.61
|
| 37 |
},
|
| 38 |
+
"previous": {
|
| 39 |
"acc": 0.952,
|
| 40 |
"ece": 0.0151,
|
| 41 |
"conf": 0.9634,
|
| 42 |
"brier": 0.0749,
|
| 43 |
"n": 1500,
|
| 44 |
+
"sec": 1.79
|
| 45 |
+
},
|
| 46 |
+
"system01": {
|
| 47 |
+
"acc": 0.9487,
|
| 48 |
+
"ece": 0.0145,
|
| 49 |
+
"conf": 0.9608,
|
| 50 |
+
"brier": 0.0785,
|
| 51 |
+
"n": 1500,
|
| 52 |
+
"sec": 1.64
|
| 53 |
}
|
| 54 |
},
|
| 55 |
"Synthetic decisions (held-out schemas)": {
|
|
|
|
| 59 |
"conf": 0.7683,
|
| 60 |
"brier": 0.5892,
|
| 61 |
"n": 1867,
|
| 62 |
+
"sec": 1.34
|
| 63 |
},
|
| 64 |
+
"previous": {
|
| 65 |
"acc": 0.8597,
|
| 66 |
"ece": 0.0241,
|
| 67 |
"conf": 0.8831,
|
| 68 |
"brier": 0.1724,
|
| 69 |
"n": 1867,
|
| 70 |
"sec": 1.35
|
| 71 |
+
},
|
| 72 |
+
"system01": {
|
| 73 |
+
"acc": 0.8548,
|
| 74 |
+
"ece": 0.031,
|
| 75 |
+
"conf": 0.8835,
|
| 76 |
+
"brier": 0.182,
|
| 77 |
+
"n": 1867,
|
| 78 |
+
"sec": 1.39
|
| 79 |
}
|
| 80 |
},
|
| 81 |
"Synthetic guard (held-out schemas)": {
|
|
|
|
| 87 |
"n": 435,
|
| 88 |
"sec": 0.36
|
| 89 |
},
|
| 90 |
+
"previous": {
|
| 91 |
"acc": 0.9678,
|
| 92 |
"ece": 0.0287,
|
| 93 |
"conf": 0.9529,
|
| 94 |
"brier": 0.031,
|
| 95 |
"n": 435,
|
| 96 |
"sec": 0.21
|
| 97 |
+
},
|
| 98 |
+
"system01": {
|
| 99 |
+
"acc": 0.9655,
|
| 100 |
+
"ece": 0.0152,
|
| 101 |
+
"conf": 0.9513,
|
| 102 |
+
"brier": 0.0307,
|
| 103 |
+
"n": 435,
|
| 104 |
+
"sec": 0.21
|
| 105 |
}
|
| 106 |
},
|
| 107 |
"Synthetic taxonomy (held-out schemas)": {
|
|
|
|
| 111 |
"conf": 0.7117,
|
| 112 |
"brier": 0.957,
|
| 113 |
"n": 1132,
|
| 114 |
+
"sec": 6.5
|
| 115 |
},
|
| 116 |
+
"previous": {
|
| 117 |
"acc": 0.8719,
|
| 118 |
"ece": 0.0607,
|
| 119 |
"conf": 0.925,
|
| 120 |
"brier": 0.1943,
|
| 121 |
"n": 1132,
|
| 122 |
+
"sec": 6.58
|
| 123 |
+
},
|
| 124 |
+
"system01": {
|
| 125 |
+
"acc": 0.871,
|
| 126 |
+
"ece": 0.0556,
|
| 127 |
+
"conf": 0.9258,
|
| 128 |
+
"brier": 0.1905,
|
| 129 |
+
"n": 1132,
|
| 130 |
+
"sec": 6.55
|
| 131 |
}
|
| 132 |
},
|
| 133 |
"Synthetic emotion (held-out schemas)": {
|
|
|
|
| 139 |
"n": 505,
|
| 140 |
"sec": 0.31
|
| 141 |
},
|
| 142 |
+
"previous": {
|
| 143 |
"acc": 0.9505,
|
| 144 |
"ece": 0.0315,
|
| 145 |
"conf": 0.9249,
|
| 146 |
"brier": 0.0799,
|
| 147 |
"n": 505,
|
| 148 |
"sec": 0.31
|
| 149 |
+
},
|
| 150 |
+
"system01": {
|
| 151 |
+
"acc": 0.9347,
|
| 152 |
+
"ece": 0.0338,
|
| 153 |
+
"conf": 0.9322,
|
| 154 |
+
"brier": 0.0837,
|
| 155 |
+
"n": 505,
|
| 156 |
+
"sec": 0.34
|
| 157 |
}
|
| 158 |
},
|
| 159 |
"AG News (4)": {
|
|
|
|
| 163 |
"conf": 0.9674,
|
| 164 |
"brier": 0.1015,
|
| 165 |
"n": 1000,
|
| 166 |
+
"sec": 0.63
|
| 167 |
},
|
| 168 |
+
"previous": {
|
| 169 |
"acc": 0.931,
|
| 170 |
"ece": 0.0247,
|
| 171 |
"conf": 0.9438,
|
| 172 |
"brier": 0.114,
|
| 173 |
"n": 1000,
|
| 174 |
+
"sec": 0.65
|
| 175 |
+
},
|
| 176 |
+
"system01": {
|
| 177 |
+
"acc": 0.931,
|
| 178 |
+
"ece": 0.0322,
|
| 179 |
+
"conf": 0.9499,
|
| 180 |
+
"brier": 0.117,
|
| 181 |
+
"n": 1000,
|
| 182 |
"sec": 0.66
|
| 183 |
}
|
| 184 |
},
|
|
|
|
| 191 |
"n": 1025,
|
| 192 |
"sec": 0.75
|
| 193 |
},
|
| 194 |
+
"previous": {
|
| 195 |
"acc": 0.4537,
|
| 196 |
"ece": 0.2924,
|
| 197 |
"conf": 0.746,
|
| 198 |
"brier": 0.7854,
|
| 199 |
"n": 1025,
|
| 200 |
"sec": 0.75
|
| 201 |
+
},
|
| 202 |
+
"system01": {
|
| 203 |
+
"acc": 0.4488,
|
| 204 |
+
"ece": 0.3378,
|
| 205 |
+
"conf": 0.7834,
|
| 206 |
+
"brier": 0.8405,
|
| 207 |
+
"n": 1025,
|
| 208 |
+
"sec": 0.76
|
| 209 |
}
|
| 210 |
},
|
| 211 |
"Banking77 (77)": {
|
|
|
|
| 215 |
"conf": 0.8734,
|
| 216 |
"brier": 0.8089,
|
| 217 |
"n": 1001,
|
| 218 |
+
"sec": 8.3
|
| 219 |
},
|
| 220 |
+
"previous": {
|
| 221 |
"acc": 0.6234,
|
| 222 |
"ece": 0.1326,
|
| 223 |
"conf": 0.756,
|
| 224 |
"brier": 0.5411,
|
| 225 |
"n": 1001,
|
| 226 |
+
"sec": 8.43
|
| 227 |
+
},
|
| 228 |
+
"system01": {
|
| 229 |
+
"acc": 0.6224,
|
| 230 |
+
"ece": 0.1458,
|
| 231 |
+
"conf": 0.7682,
|
| 232 |
+
"brier": 0.5425,
|
| 233 |
+
"n": 1001,
|
| 234 |
+
"sec": 8.37
|
| 235 |
}
|
| 236 |
},
|
| 237 |
"SMS spam (yes/no)": {
|
|
|
|
| 241 |
"conf": 0.9435,
|
| 242 |
"brier": 0.4624,
|
| 243 |
"n": 1000,
|
| 244 |
+
"sec": 0.52
|
| 245 |
},
|
| 246 |
+
"previous": {
|
| 247 |
"acc": 0.936,
|
| 248 |
"ece": 0.0387,
|
| 249 |
"conf": 0.9057,
|
| 250 |
"brier": 0.095,
|
| 251 |
"n": 1000,
|
| 252 |
+
"sec": 0.49
|
| 253 |
+
},
|
| 254 |
+
"system01": {
|
| 255 |
+
"acc": 0.936,
|
| 256 |
+
"ece": 0.023,
|
| 257 |
+
"conf": 0.9176,
|
| 258 |
+
"brier": 0.0985,
|
| 259 |
+
"n": 1000,
|
| 260 |
+
"sec": 0.64
|
| 261 |
}
|
| 262 |
},
|
| 263 |
"AZ app-review sentiment": {
|
|
|
|
| 269 |
"n": 1000,
|
| 270 |
"sec": 0.37
|
| 271 |
},
|
| 272 |
+
"previous": {
|
| 273 |
"acc": 0.857,
|
| 274 |
"ece": 0.1112,
|
| 275 |
"conf": 0.7462,
|
| 276 |
"brier": 0.2424,
|
| 277 |
"n": 1000,
|
| 278 |
"sec": 0.53
|
| 279 |
+
},
|
| 280 |
+
"system01": {
|
| 281 |
+
"acc": 0.849,
|
| 282 |
+
"ece": 0.0907,
|
| 283 |
+
"conf": 0.7682,
|
| 284 |
+
"brier": 0.239,
|
| 285 |
+
"n": 1000,
|
| 286 |
+
"sec": 0.38
|
| 287 |
}
|
| 288 |
},
|
| 289 |
"MASSIVE-intent@20 (az)": {
|
|
|
|
| 293 |
"conf": 0.6292,
|
| 294 |
"brier": 0.8902,
|
| 295 |
"n": 1000,
|
| 296 |
+
"sec": 1.54
|
| 297 |
},
|
| 298 |
+
"previous": {
|
| 299 |
"acc": 0.898,
|
| 300 |
"ece": 0.0459,
|
| 301 |
"conf": 0.858,
|
| 302 |
"brier": 0.1482,
|
| 303 |
"n": 1000,
|
| 304 |
+
"sec": 1.56
|
| 305 |
+
},
|
| 306 |
+
"system01": {
|
| 307 |
+
"acc": 0.894,
|
| 308 |
+
"ece": 0.047,
|
| 309 |
+
"conf": 0.8676,
|
| 310 |
+
"brier": 0.1527,
|
| 311 |
+
"n": 1000,
|
| 312 |
+
"sec": 1.53
|
| 313 |
}
|
| 314 |
},
|
| 315 |
"Intent yes/no (az)": {
|
|
|
|
| 321 |
"n": 1000,
|
| 322 |
"sec": 0.35
|
| 323 |
},
|
| 324 |
+
"previous": {
|
| 325 |
"acc": 0.944,
|
| 326 |
"ece": 0.0161,
|
| 327 |
"conf": 0.9465,
|
| 328 |
"brier": 0.0389,
|
| 329 |
"n": 1000,
|
| 330 |
+
"sec": 0.36
|
| 331 |
+
},
|
| 332 |
+
"system01": {
|
| 333 |
+
"acc": 0.951,
|
| 334 |
+
"ece": 0.0159,
|
| 335 |
+
"conf": 0.9471,
|
| 336 |
+
"brier": 0.0377,
|
| 337 |
+
"n": 1000,
|
| 338 |
+
"sec": 0.36
|
| 339 |
}
|
| 340 |
},
|
| 341 |
"MASSIVE-intent@20 (en)": {
|
|
|
|
| 345 |
"conf": 0.7832,
|
| 346 |
"brier": 0.5463,
|
| 347 |
"n": 1000,
|
| 348 |
+
"sec": 1.49
|
| 349 |
},
|
| 350 |
+
"previous": {
|
| 351 |
"acc": 0.932,
|
| 352 |
"ece": 0.0563,
|
| 353 |
"conf": 0.8768,
|
| 354 |
"brier": 0.1142,
|
| 355 |
"n": 1000,
|
| 356 |
+
"sec": 1.52
|
| 357 |
+
},
|
| 358 |
+
"system01": {
|
| 359 |
+
"acc": 0.93,
|
| 360 |
+
"ece": 0.0453,
|
| 361 |
+
"conf": 0.8887,
|
| 362 |
+
"brier": 0.1108,
|
| 363 |
+
"n": 1000,
|
| 364 |
+
"sec": 1.5
|
| 365 |
}
|
| 366 |
},
|
| 367 |
"Intent yes/no (en)": {
|
|
|
|
| 373 |
"n": 1000,
|
| 374 |
"sec": 0.47
|
| 375 |
},
|
| 376 |
+
"previous": {
|
| 377 |
"acc": 0.956,
|
| 378 |
"ece": 0.0183,
|
| 379 |
"conf": 0.9431,
|
| 380 |
"brier": 0.0324,
|
| 381 |
"n": 1000,
|
| 382 |
"sec": 0.32
|
| 383 |
+
},
|
| 384 |
+
"system01": {
|
| 385 |
+
"acc": 0.954,
|
| 386 |
+
"ece": 0.0199,
|
| 387 |
+
"conf": 0.9412,
|
| 388 |
+
"brier": 0.0351,
|
| 389 |
+
"n": 1000,
|
| 390 |
+
"sec": 0.32
|
| 391 |
}
|
| 392 |
},
|
| 393 |
"MASSIVE-scenario (en)": {
|
|
|
|
| 397 |
"conf": 0.777,
|
| 398 |
"brier": 0.4383,
|
| 399 |
"n": 500,
|
| 400 |
+
"sec": 0.73
|
| 401 |
},
|
| 402 |
+
"previous": {
|
| 403 |
"acc": 0.91,
|
| 404 |
"ece": 0.0651,
|
| 405 |
"conf": 0.8491,
|
| 406 |
"brier": 0.1413,
|
| 407 |
"n": 500,
|
| 408 |
"sec": 0.74
|
| 409 |
+
},
|
| 410 |
+
"system01": {
|
| 411 |
+
"acc": 0.91,
|
| 412 |
+
"ece": 0.0411,
|
| 413 |
+
"conf": 0.8751,
|
| 414 |
+
"brier": 0.1416,
|
| 415 |
+
"n": 500,
|
| 416 |
+
"sec": 0.75
|
| 417 |
}
|
| 418 |
},
|
| 419 |
"MASSIVE-scenario (az)": {
|
|
|
|
| 423 |
"conf": 0.6237,
|
| 424 |
"brier": 0.7993,
|
| 425 |
"n": 500,
|
| 426 |
+
"sec": 0.76
|
| 427 |
},
|
| 428 |
+
"previous": {
|
| 429 |
"acc": 0.884,
|
| 430 |
"ece": 0.0527,
|
| 431 |
"conf": 0.8509,
|
| 432 |
"brier": 0.1745,
|
| 433 |
"n": 500,
|
| 434 |
+
"sec": 0.78
|
| 435 |
+
},
|
| 436 |
+
"system01": {
|
| 437 |
+
"acc": 0.882,
|
| 438 |
+
"ece": 0.0644,
|
| 439 |
+
"conf": 0.8704,
|
| 440 |
+
"brier": 0.1752,
|
| 441 |
+
"n": 500,
|
| 442 |
"sec": 0.77
|
| 443 |
}
|
| 444 |
},
|
|
|
|
| 449 |
"conf": 0.734,
|
| 450 |
"brier": 0.6105,
|
| 451 |
"n": 500,
|
| 452 |
+
"sec": 0.74
|
| 453 |
},
|
| 454 |
+
"previous": {
|
| 455 |
"acc": 0.886,
|
| 456 |
"ece": 0.059,
|
| 457 |
"conf": 0.8326,
|
| 458 |
"brier": 0.1712,
|
| 459 |
"n": 500,
|
| 460 |
+
"sec": 0.76
|
| 461 |
+
},
|
| 462 |
+
"system01": {
|
| 463 |
+
"acc": 0.906,
|
| 464 |
+
"ece": 0.0515,
|
| 465 |
+
"conf": 0.8673,
|
| 466 |
+
"brier": 0.1574,
|
| 467 |
+
"n": 500,
|
| 468 |
+
"sec": 0.76
|
| 469 |
}
|
| 470 |
},
|
| 471 |
"MASSIVE-scenario (tr)": {
|
|
|
|
| 475 |
"conf": 0.6785,
|
| 476 |
"brier": 0.7032,
|
| 477 |
"n": 500,
|
| 478 |
+
"sec": 0.73
|
| 479 |
},
|
| 480 |
+
"previous": {
|
| 481 |
"acc": 0.886,
|
| 482 |
"ece": 0.0729,
|
| 483 |
"conf": 0.8287,
|
| 484 |
"brier": 0.1687,
|
| 485 |
"n": 500,
|
| 486 |
+
"sec": 0.76
|
| 487 |
+
},
|
| 488 |
+
"system01": {
|
| 489 |
+
"acc": 0.882,
|
| 490 |
+
"ece": 0.0559,
|
| 491 |
+
"conf": 0.8566,
|
| 492 |
+
"brier": 0.1814,
|
| 493 |
+
"n": 500,
|
| 494 |
"sec": 0.75
|
| 495 |
}
|
| 496 |
},
|
|
|
|
| 501 |
"conf": 0.7256,
|
| 502 |
"brier": 0.6269,
|
| 503 |
"n": 500,
|
| 504 |
+
"sec": 0.74
|
| 505 |
},
|
| 506 |
+
"previous": {
|
| 507 |
"acc": 0.884,
|
| 508 |
"ece": 0.057,
|
| 509 |
"conf": 0.8343,
|
| 510 |
"brier": 0.1739,
|
| 511 |
"n": 500,
|
| 512 |
+
"sec": 0.76
|
| 513 |
+
},
|
| 514 |
+
"system01": {
|
| 515 |
+
"acc": 0.868,
|
| 516 |
+
"ece": 0.046,
|
| 517 |
+
"conf": 0.859,
|
| 518 |
+
"brier": 0.1761,
|
| 519 |
+
"n": 500,
|
| 520 |
"sec": 0.75
|
| 521 |
}
|
| 522 |
},
|
|
|
|
| 527 |
"conf": 0.738,
|
| 528 |
"brier": 0.5843,
|
| 529 |
"n": 500,
|
| 530 |
+
"sec": 0.75
|
| 531 |
},
|
| 532 |
+
"previous": {
|
| 533 |
"acc": 0.894,
|
| 534 |
"ece": 0.0566,
|
| 535 |
"conf": 0.8392,
|
| 536 |
"brier": 0.1544,
|
| 537 |
"n": 500,
|
| 538 |
"sec": 0.75
|
| 539 |
+
},
|
| 540 |
+
"system01": {
|
| 541 |
+
"acc": 0.906,
|
| 542 |
+
"ece": 0.052,
|
| 543 |
+
"conf": 0.8681,
|
| 544 |
+
"brier": 0.1445,
|
| 545 |
+
"n": 500,
|
| 546 |
+
"sec": 0.76
|
| 547 |
}
|
| 548 |
},
|
| 549 |
"MASSIVE-scenario (es)": {
|
|
|
|
| 553 |
"conf": 0.7339,
|
| 554 |
"brier": 0.6289,
|
| 555 |
"n": 500,
|
| 556 |
+
"sec": 0.74
|
| 557 |
},
|
| 558 |
+
"previous": {
|
| 559 |
"acc": 0.878,
|
| 560 |
"ece": 0.0574,
|
| 561 |
"conf": 0.8233,
|
| 562 |
"brier": 0.1851,
|
| 563 |
"n": 500,
|
| 564 |
+
"sec": 0.75
|
| 565 |
+
},
|
| 566 |
+
"system01": {
|
| 567 |
+
"acc": 0.878,
|
| 568 |
+
"ece": 0.0376,
|
| 569 |
+
"conf": 0.8527,
|
| 570 |
+
"brier": 0.1781,
|
| 571 |
+
"n": 500,
|
| 572 |
+
"sec": 0.74
|
| 573 |
}
|
| 574 |
},
|
| 575 |
"MASSIVE-scenario (ar)": {
|
|
|
|
| 579 |
"conf": 0.6603,
|
| 580 |
"brier": 0.7943,
|
| 581 |
"n": 500,
|
| 582 |
+
"sec": 0.74
|
| 583 |
},
|
| 584 |
+
"previous": {
|
| 585 |
"acc": 0.828,
|
| 586 |
"ece": 0.0618,
|
| 587 |
"conf": 0.8009,
|
| 588 |
"brier": 0.2499,
|
| 589 |
"n": 500,
|
| 590 |
+
"sec": 0.77
|
| 591 |
+
},
|
| 592 |
+
"system01": {
|
| 593 |
+
"acc": 0.818,
|
| 594 |
+
"ece": 0.0442,
|
| 595 |
+
"conf": 0.8322,
|
| 596 |
+
"brier": 0.2628,
|
| 597 |
+
"n": 500,
|
| 598 |
+
"sec": 0.75
|
| 599 |
}
|
| 600 |
},
|
| 601 |
"MASSIVE-scenario (hi)": {
|
|
|
|
| 607 |
"n": 500,
|
| 608 |
"sec": 0.76
|
| 609 |
},
|
| 610 |
+
"previous": {
|
| 611 |
"acc": 0.86,
|
| 612 |
"ece": 0.0466,
|
| 613 |
"conf": 0.8197,
|
| 614 |
"brier": 0.2053,
|
| 615 |
"n": 500,
|
| 616 |
"sec": 0.77
|
| 617 |
+
},
|
| 618 |
+
"system01": {
|
| 619 |
+
"acc": 0.846,
|
| 620 |
+
"ece": 0.0423,
|
| 621 |
+
"conf": 0.8405,
|
| 622 |
+
"brier": 0.2133,
|
| 623 |
+
"n": 500,
|
| 624 |
+
"sec": 0.77
|
| 625 |
}
|
| 626 |
},
|
| 627 |
"MASSIVE-scenario (zh-CN)": {
|
|
|
|
| 633 |
"n": 500,
|
| 634 |
"sec": 0.74
|
| 635 |
},
|
| 636 |
+
"previous": {
|
| 637 |
"acc": 0.872,
|
| 638 |
"ece": 0.0647,
|
| 639 |
"conf": 0.8212,
|
| 640 |
"brier": 0.1766,
|
| 641 |
"n": 500,
|
| 642 |
"sec": 0.74
|
| 643 |
+
},
|
| 644 |
+
"system01": {
|
| 645 |
+
"acc": 0.87,
|
| 646 |
+
"ece": 0.0518,
|
| 647 |
+
"conf": 0.8568,
|
| 648 |
+
"brier": 0.179,
|
| 649 |
+
"n": 500,
|
| 650 |
+
"sec": 0.74
|
| 651 |
}
|
| 652 |
},
|
| 653 |
"MASSIVE-scenario (fa)": {
|
|
|
|
| 657 |
"conf": 0.6965,
|
| 658 |
"brier": 0.7663,
|
| 659 |
"n": 500,
|
| 660 |
+
"sec": 0.75
|
| 661 |
},
|
| 662 |
+
"previous": {
|
| 663 |
"acc": 0.876,
|
| 664 |
"ece": 0.0617,
|
| 665 |
"conf": 0.8308,
|
| 666 |
"brier": 0.1695,
|
| 667 |
"n": 500,
|
| 668 |
+
"sec": 0.75
|
| 669 |
+
},
|
| 670 |
+
"system01": {
|
| 671 |
+
"acc": 0.876,
|
| 672 |
+
"ece": 0.0476,
|
| 673 |
+
"conf": 0.8541,
|
| 674 |
+
"brier": 0.1747,
|
| 675 |
+
"n": 500,
|
| 676 |
"sec": 0.76
|
| 677 |
}
|
| 678 |
},
|
|
|
|
| 683 |
"conf": 0.4054,
|
| 684 |
"brier": 1.036,
|
| 685 |
"n": 500,
|
| 686 |
+
"sec": 0.78
|
| 687 |
},
|
| 688 |
+
"previous": {
|
| 689 |
"acc": 0.79,
|
| 690 |
"ece": 0.0572,
|
| 691 |
"conf": 0.7382,
|
| 692 |
"brier": 0.2991,
|
| 693 |
"n": 500,
|
| 694 |
+
"sec": 0.79
|
| 695 |
+
},
|
| 696 |
+
"system01": {
|
| 697 |
+
"acc": 0.774,
|
| 698 |
+
"ece": 0.0412,
|
| 699 |
+
"conf": 0.7722,
|
| 700 |
+
"brier": 0.3159,
|
| 701 |
+
"n": 500,
|
| 702 |
"sec": 0.8
|
| 703 |
}
|
| 704 |
},
|
|
|
|
| 709 |
"conf": 0.7711,
|
| 710 |
"brier": 0.6096,
|
| 711 |
"n": 500,
|
| 712 |
+
"sec": 0.74
|
| 713 |
},
|
| 714 |
+
"previous": {
|
| 715 |
"acc": 0.898,
|
| 716 |
"ece": 0.0546,
|
| 717 |
"conf": 0.8542,
|
| 718 |
"brier": 0.1455,
|
| 719 |
"n": 500,
|
| 720 |
+
"sec": 0.73
|
| 721 |
+
},
|
| 722 |
+
"system01": {
|
| 723 |
+
"acc": 0.906,
|
| 724 |
+
"ece": 0.0424,
|
| 725 |
+
"conf": 0.8843,
|
| 726 |
+
"brier": 0.1424,
|
| 727 |
+
"n": 500,
|
| 728 |
+
"sec": 0.75
|
| 729 |
}
|
| 730 |
},
|
| 731 |
"MASSIVE-scenario (ko)": {
|
|
|
|
| 735 |
"conf": 0.7026,
|
| 736 |
"brier": 0.7354,
|
| 737 |
"n": 500,
|
| 738 |
+
"sec": 0.75
|
| 739 |
},
|
| 740 |
+
"previous": {
|
| 741 |
"acc": 0.878,
|
| 742 |
"ece": 0.0532,
|
| 743 |
"conf": 0.8363,
|
| 744 |
"brier": 0.1827,
|
| 745 |
"n": 500,
|
| 746 |
+
"sec": 0.76
|
| 747 |
+
},
|
| 748 |
+
"system01": {
|
| 749 |
+
"acc": 0.872,
|
| 750 |
+
"ece": 0.0505,
|
| 751 |
+
"conf": 0.8631,
|
| 752 |
+
"brier": 0.1871,
|
| 753 |
+
"n": 500,
|
| 754 |
+
"sec": 0.76
|
| 755 |
}
|
| 756 |
},
|
| 757 |
"SIB-200 (az)": {
|
|
|
|
| 761 |
"conf": 0.858,
|
| 762 |
"brier": 0.5221,
|
| 763 |
"n": 204,
|
| 764 |
+
"sec": 0.18
|
| 765 |
},
|
| 766 |
+
"previous": {
|
| 767 |
"acc": 0.7451,
|
| 768 |
"ece": 0.1092,
|
| 769 |
"conf": 0.8319,
|
| 770 |
"brier": 0.3752,
|
| 771 |
"n": 204,
|
| 772 |
+
"sec": 0.19
|
| 773 |
+
},
|
| 774 |
+
"system01": {
|
| 775 |
+
"acc": 0.7402,
|
| 776 |
+
"ece": 0.1377,
|
| 777 |
+
"conf": 0.8616,
|
| 778 |
+
"brier": 0.3965,
|
| 779 |
+
"n": 204,
|
| 780 |
"sec": 0.18
|
| 781 |
}
|
| 782 |
},
|
|
|
|
| 789 |
"n": 204,
|
| 790 |
"sec": 0.15
|
| 791 |
},
|
| 792 |
+
"previous": {
|
| 793 |
"acc": 0.8529,
|
| 794 |
"ece": 0.0799,
|
| 795 |
"conf": 0.802,
|
| 796 |
"brier": 0.261,
|
| 797 |
"n": 204,
|
| 798 |
"sec": 0.15
|
| 799 |
+
},
|
| 800 |
+
"system01": {
|
| 801 |
+
"acc": 0.848,
|
| 802 |
+
"ece": 0.0838,
|
| 803 |
+
"conf": 0.8333,
|
| 804 |
+
"brier": 0.2507,
|
| 805 |
+
"n": 204,
|
| 806 |
+
"sec": 0.15
|
| 807 |
}
|
| 808 |
},
|
| 809 |
"SIB-200 (ru)": {
|
|
|
|
| 815 |
"n": 204,
|
| 816 |
"sec": 0.16
|
| 817 |
},
|
| 818 |
+
"previous": {
|
| 819 |
"acc": 0.7598,
|
| 820 |
"ece": 0.0736,
|
| 821 |
"conf": 0.8153,
|
| 822 |
"brier": 0.3231,
|
| 823 |
"n": 204,
|
| 824 |
"sec": 0.17
|
| 825 |
+
},
|
| 826 |
+
"system01": {
|
| 827 |
+
"acc": 0.7843,
|
| 828 |
+
"ece": 0.0939,
|
| 829 |
+
"conf": 0.8449,
|
| 830 |
+
"brier": 0.3221,
|
| 831 |
+
"n": 204,
|
| 832 |
+
"sec": 0.16
|
| 833 |
}
|
| 834 |
},
|
| 835 |
"SIB-200 (tr)": {
|
|
|
|
| 839 |
"conf": 0.8547,
|
| 840 |
"brier": 0.4185,
|
| 841 |
"n": 204,
|
| 842 |
+
"sec": 0.17
|
| 843 |
},
|
| 844 |
+
"previous": {
|
| 845 |
"acc": 0.7696,
|
| 846 |
"ece": 0.1237,
|
| 847 |
"conf": 0.8294,
|
| 848 |
"brier": 0.3642,
|
| 849 |
"n": 204,
|
| 850 |
"sec": 0.16
|
| 851 |
+
},
|
| 852 |
+
"system01": {
|
| 853 |
+
"acc": 0.7696,
|
| 854 |
+
"ece": 0.1289,
|
| 855 |
+
"conf": 0.8515,
|
| 856 |
+
"brier": 0.3684,
|
| 857 |
+
"n": 204,
|
| 858 |
+
"sec": 0.16
|
| 859 |
}
|
| 860 |
},
|
| 861 |
"SIB-200 (de)": {
|
|
|
|
| 867 |
"n": 204,
|
| 868 |
"sec": 0.16
|
| 869 |
},
|
| 870 |
+
"previous": {
|
| 871 |
"acc": 0.8039,
|
| 872 |
"ece": 0.0818,
|
| 873 |
"conf": 0.8238,
|
| 874 |
"brier": 0.293,
|
| 875 |
"n": 204,
|
| 876 |
"sec": 0.16
|
| 877 |
+
},
|
| 878 |
+
"system01": {
|
| 879 |
+
"acc": 0.8186,
|
| 880 |
+
"ece": 0.095,
|
| 881 |
+
"conf": 0.8516,
|
| 882 |
+
"brier": 0.2808,
|
| 883 |
+
"n": 204,
|
| 884 |
+
"sec": 0.16
|
| 885 |
}
|
| 886 |
},
|
| 887 |
"SIB-200 (ar)": {
|
|
|
|
| 891 |
"conf": 0.8589,
|
| 892 |
"brier": 0.4077,
|
| 893 |
"n": 204,
|
| 894 |
+
"sec": 0.17
|
| 895 |
},
|
| 896 |
+
"previous": {
|
| 897 |
"acc": 0.7745,
|
| 898 |
"ece": 0.0704,
|
| 899 |
"conf": 0.8245,
|
| 900 |
"brier": 0.333,
|
| 901 |
"n": 204,
|
| 902 |
"sec": 0.17
|
| 903 |
+
},
|
| 904 |
+
"system01": {
|
| 905 |
+
"acc": 0.7794,
|
| 906 |
+
"ece": 0.0872,
|
| 907 |
+
"conf": 0.8523,
|
| 908 |
+
"brier": 0.332,
|
| 909 |
+
"n": 204,
|
| 910 |
+
"sec": 0.16
|
| 911 |
}
|
| 912 |
},
|
| 913 |
"SIB-200 (hi)": {
|
|
|
|
| 917 |
"conf": 0.8433,
|
| 918 |
"brier": 0.4835,
|
| 919 |
"n": 204,
|
| 920 |
+
"sec": 0.17
|
| 921 |
},
|
| 922 |
+
"previous": {
|
| 923 |
"acc": 0.7157,
|
| 924 |
"ece": 0.0998,
|
| 925 |
"conf": 0.8112,
|
| 926 |
"brier": 0.4172,
|
| 927 |
"n": 204,
|
| 928 |
"sec": 0.18
|
| 929 |
+
},
|
| 930 |
+
"system01": {
|
| 931 |
+
"acc": 0.7157,
|
| 932 |
+
"ece": 0.1389,
|
| 933 |
+
"conf": 0.8405,
|
| 934 |
+
"brier": 0.4159,
|
| 935 |
+
"n": 204,
|
| 936 |
+
"sec": 0.17
|
| 937 |
}
|
| 938 |
},
|
| 939 |
"SIB-200 (zh)": {
|
|
|
|
| 945 |
"n": 204,
|
| 946 |
"sec": 0.16
|
| 947 |
},
|
| 948 |
+
"previous": {
|
| 949 |
"acc": 0.8039,
|
| 950 |
"ece": 0.0891,
|
| 951 |
"conf": 0.818,
|
| 952 |
"brier": 0.3114,
|
| 953 |
"n": 204,
|
| 954 |
+
"sec": 0.15
|
| 955 |
+
},
|
| 956 |
+
"system01": {
|
| 957 |
+
"acc": 0.7941,
|
| 958 |
+
"ece": 0.101,
|
| 959 |
+
"conf": 0.8479,
|
| 960 |
+
"brier": 0.3099,
|
| 961 |
+
"n": 204,
|
| 962 |
+
"sec": 0.15
|
| 963 |
}
|
| 964 |
},
|
| 965 |
"SIB-200 (kk)": {
|
|
|
|
| 969 |
"conf": 0.815,
|
| 970 |
"brier": 0.4644,
|
| 971 |
"n": 204,
|
| 972 |
+
"sec": 0.18
|
| 973 |
},
|
| 974 |
+
"previous": {
|
| 975 |
"acc": 0.7402,
|
| 976 |
"ece": 0.0659,
|
| 977 |
"conf": 0.7902,
|
| 978 |
"brier": 0.3776,
|
| 979 |
"n": 204,
|
| 980 |
+
"sec": 0.18
|
| 981 |
+
},
|
| 982 |
+
"system01": {
|
| 983 |
+
"acc": 0.75,
|
| 984 |
+
"ece": 0.0964,
|
| 985 |
+
"conf": 0.823,
|
| 986 |
+
"brier": 0.3769,
|
| 987 |
+
"n": 204,
|
| 988 |
"sec": 0.19
|
| 989 |
}
|
| 990 |
},
|
|
|
|
| 995 |
"conf": 0.8053,
|
| 996 |
"brier": 0.6262,
|
| 997 |
"n": 204,
|
| 998 |
+
"sec": 0.17
|
| 999 |
},
|
| 1000 |
+
"previous": {
|
| 1001 |
"acc": 0.6667,
|
| 1002 |
"ece": 0.1444,
|
| 1003 |
"conf": 0.7902,
|
| 1004 |
"brier": 0.4792,
|
| 1005 |
"n": 204,
|
| 1006 |
"sec": 0.18
|
| 1007 |
+
},
|
| 1008 |
+
"system01": {
|
| 1009 |
+
"acc": 0.6569,
|
| 1010 |
+
"ece": 0.1702,
|
| 1011 |
+
"conf": 0.8203,
|
| 1012 |
+
"brier": 0.4979,
|
| 1013 |
+
"n": 204,
|
| 1014 |
+
"sec": 0.18
|
| 1015 |
}
|
| 1016 |
}
|
| 1017 |
},
|
results/reliability.json
CHANGED
|
@@ -51,51 +51,51 @@
|
|
| 51 |
"acc": 0.4400206825232678,
|
| 52 |
"n": 5802
|
| 53 |
},
|
| 54 |
-
"
|
| 55 |
"bins": [
|
| 56 |
{
|
| 57 |
-
"conf": 0.
|
| 58 |
-
"acc": 0.
|
| 59 |
-
"n":
|
| 60 |
},
|
| 61 |
{
|
| 62 |
-
"conf": 0.
|
| 63 |
-
"acc": 0.
|
| 64 |
-
"n":
|
| 65 |
},
|
| 66 |
{
|
| 67 |
-
"conf": 0.
|
| 68 |
-
"acc": 0.
|
| 69 |
-
"n":
|
| 70 |
},
|
| 71 |
{
|
| 72 |
-
"conf": 0.
|
| 73 |
-
"acc": 0.
|
| 74 |
-
"n":
|
| 75 |
},
|
| 76 |
{
|
| 77 |
-
"conf": 0.
|
| 78 |
-
"acc": 0.
|
| 79 |
-
"n":
|
| 80 |
},
|
| 81 |
{
|
| 82 |
-
"conf": 0.
|
| 83 |
-
"acc": 0.
|
| 84 |
-
"n":
|
| 85 |
},
|
| 86 |
{
|
| 87 |
-
"conf": 0.
|
| 88 |
-
"acc": 0.
|
| 89 |
-
"n":
|
| 90 |
},
|
| 91 |
{
|
| 92 |
-
"conf": 0.
|
| 93 |
-
"acc": 0.
|
| 94 |
-
"n":
|
| 95 |
}
|
| 96 |
],
|
| 97 |
-
"ece": 0.
|
| 98 |
-
"acc": 0.
|
| 99 |
"n": 5802
|
| 100 |
}
|
| 101 |
}
|
|
|
|
| 51 |
"acc": 0.4400206825232678,
|
| 52 |
"n": 5802
|
| 53 |
},
|
| 54 |
+
"system01": {
|
| 55 |
"bins": [
|
| 56 |
{
|
| 57 |
+
"conf": 0.2558885714285714,
|
| 58 |
+
"acc": 0.2,
|
| 59 |
+
"n": 35
|
| 60 |
},
|
| 61 |
{
|
| 62 |
+
"conf": 0.35227183098591547,
|
| 63 |
+
"acc": 0.38028169014084506,
|
| 64 |
+
"n": 71
|
| 65 |
},
|
| 66 |
{
|
| 67 |
+
"conf": 0.45390079365079355,
|
| 68 |
+
"acc": 0.4523809523809524,
|
| 69 |
+
"n": 126
|
| 70 |
},
|
| 71 |
{
|
| 72 |
+
"conf": 0.5491653284671533,
|
| 73 |
+
"acc": 0.5145985401459854,
|
| 74 |
+
"n": 274
|
| 75 |
},
|
| 76 |
{
|
| 77 |
+
"conf": 0.6557608465608464,
|
| 78 |
+
"acc": 0.6851851851851852,
|
| 79 |
+
"n": 378
|
| 80 |
},
|
| 81 |
{
|
| 82 |
+
"conf": 0.7575994208494209,
|
| 83 |
+
"acc": 0.7644787644787645,
|
| 84 |
+
"n": 518
|
| 85 |
},
|
| 86 |
{
|
| 87 |
+
"conf": 0.8545899481865286,
|
| 88 |
+
"acc": 0.8673575129533678,
|
| 89 |
+
"n": 965
|
| 90 |
},
|
| 91 |
{
|
| 92 |
+
"conf": 0.9698653733955659,
|
| 93 |
+
"acc": 0.9775379229871646,
|
| 94 |
+
"n": 3428
|
| 95 |
}
|
| 96 |
],
|
| 97 |
+
"ece": 0.015023319544984481,
|
| 98 |
+
"acc": 0.8748707342295761,
|
| 99 |
"n": 5802
|
| 100 |
}
|
| 101 |
}
|
results/train_curve_v5.jsonl
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"step": 50, "loss": -0.46460499793291093, "acc": 0.9929770278930664}
|
| 2 |
+
{"step": 100, "loss": -0.45240239225327966, "acc": 0.9917954707145691}
|
| 3 |
+
{"step": 150, "loss": -0.46087496548891066, "acc": 0.9941230618953705}
|
| 4 |
+
{"step": 200, "loss": -0.46089778661727904, "acc": 0.9919860935211182}
|
| 5 |
+
{"step": 250, "loss": -0.4543023282289505, "acc": 0.989357796907425}
|
| 6 |
+
{"step": 300, "loss": -0.44895768523216245, "acc": 0.991442221403122}
|
| 7 |
+
{"step": 350, "loss": -0.44892375230789183, "acc": 0.9897629582881927}
|
| 8 |
+
{"step": 400, "loss": -0.43916955679655073, "acc": 0.9877720475196838}
|
| 9 |
+
{"step": 450, "loss": -0.4417329607158899, "acc": 0.989140590429306}
|
| 10 |
+
{"step": 500, "loss": -0.41205301411449907, "acc": 0.9864362573623657}
|
| 11 |
+
{"step": 550, "loss": -0.4322704815864563, "acc": 0.9885031306743621}
|
| 12 |
+
{"step": 600, "loss": -0.44779003083705904, "acc": 0.9887004089355469}
|
| 13 |
+
{"step": 650, "loss": -0.43306734576821326, "acc": 0.988046680688858}
|
| 14 |
+
{"step": 700, "loss": -0.44677913799881935, "acc": 0.9909416508674621}
|
| 15 |
+
{"step": 750, "loss": -0.455992049574852, "acc": 0.9912621557712555}
|
rl_agent_config.json
CHANGED
|
@@ -9,18 +9,18 @@
|
|
| 9 |
},
|
| 10 |
"cost_wrong_act": 3.0,
|
| 11 |
"amp_dtype": "bf16",
|
| 12 |
-
"model_name": "
|
| 13 |
"temperature": [
|
| 14 |
-
3.
|
| 15 |
-
|
| 16 |
-
2.
|
| 17 |
],
|
| 18 |
"temperature_by_options": {
|
| 19 |
-
"noul:2": 2.
|
| 20 |
-
"choice:3-5": 3.
|
| 21 |
-
"choice:6-10":
|
| 22 |
-
"score:3-5":
|
| 23 |
-
"choice:11+": 3.
|
| 24 |
},
|
| 25 |
"auto_head_budget": true,
|
| 26 |
"tournament_above": 24,
|
|
@@ -29,13 +29,11 @@
|
|
| 29 |
"epochs": 2,
|
| 30 |
"steps": 5468,
|
| 31 |
"items": 157912,
|
| 32 |
-
"minutes_on_rtx4090": 14.4,
|
| 33 |
"languages": "az, en, ru, tr, de, fr, es, ar, hi, zh, fa, ka, ja, ko, uk, kk, uz, it",
|
| 34 |
"held_out_for_evaluation": "SIB-200, AG News, DAIR Emotion, Banking77, SMS spam, hand-written routing set, 5% of synthetic schemas",
|
| 35 |
"post_training": {
|
| 36 |
-
"
|
| 37 |
-
"
|
| 38 |
-
"minutes_on_rtx4090": 2.2
|
| 39 |
}
|
| 40 |
}
|
| 41 |
}
|
|
|
|
| 9 |
},
|
| 10 |
"cost_wrong_act": 3.0,
|
| 11 |
"amp_dtype": "bf16",
|
| 12 |
+
"model_name": "System01",
|
| 13 |
"temperature": [
|
| 14 |
+
3.7,
|
| 15 |
+
4.4,
|
| 16 |
+
2.7
|
| 17 |
],
|
| 18 |
"temperature_by_options": {
|
| 19 |
+
"noul:2": 2.7,
|
| 20 |
+
"choice:3-5": 3.7,
|
| 21 |
+
"choice:6-10": 3.5,
|
| 22 |
+
"score:3-5": 4.4,
|
| 23 |
+
"choice:11+": 3.7
|
| 24 |
},
|
| 25 |
"auto_head_budget": true,
|
| 26 |
"tournament_above": 24,
|
|
|
|
| 29 |
"epochs": 2,
|
| 30 |
"steps": 5468,
|
| 31 |
"items": 157912,
|
|
|
|
| 32 |
"languages": "az, en, ru, tr, de, fr, es, ar, hi, zh, fa, ka, ja, ko, uk, kk, uz, it",
|
| 33 |
"held_out_for_evaluation": "SIB-200, AG News, DAIR Emotion, Banking77, SMS spam, hand-written routing set, 5% of synthetic schemas",
|
| 34 |
"post_training": {
|
| 35 |
+
"stage_2": "spatial-control (procedural camera/cursor/drone/robot/car tasks, no game data), 843 steps",
|
| 36 |
+
"stage_3": "LLM/agent-routing emphasis with replay, 770 steps"
|
|
|
|
| 37 |
}
|
| 38 |
}
|
| 39 |
}
|
{system1 → system01}/__init__.py
RENAMED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
"""
|
| 2 |
|
| 3 |
from .agent import Agent, RLAgent, load
|
| 4 |
from .common import (
|
|
|
|
| 1 |
+
"""System01: fast, non-autoregressive decision engine with calibrated probabilities."""
|
| 2 |
|
| 3 |
from .agent import Agent, RLAgent, load
|
| 4 |
from .common import (
|
{system1 → system01}/agent.py
RENAMED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
"""High-level inference runtime for
|
| 2 |
import json
|
| 3 |
import os
|
| 4 |
import warnings
|
|
@@ -102,16 +102,16 @@ class Agent:
|
|
| 102 |
|
| 103 |
def __init__(
|
| 104 |
self,
|
| 105 |
-
model_id_or_path: str = "doofz/
|
| 106 |
device: Optional[str] = None,
|
| 107 |
token: Optional[str] = None,
|
| 108 |
subfolder: Optional[str] = None,
|
| 109 |
half: bool = True,
|
| 110 |
):
|
| 111 |
-
"""Load a
|
| 112 |
|
| 113 |
`subfolder` selects one checkpoint from a repo that bundles several, e.g.
|
| 114 |
-
`Agent("doofz/
|
| 115 |
downloaded, so bundling does not cost every user the whole family.
|
| 116 |
"""
|
| 117 |
from safetensors.torch import load_file
|
|
@@ -132,7 +132,7 @@ class Agent:
|
|
| 132 |
kw = {
|
| 133 |
"token": token or os.environ.get("HF_TOKEN"),
|
| 134 |
"allow_patterns": [prefix + name for name in (
|
| 135 |
-
"rl_agent_config.json", "model.safetensors", "tokenizer/*", "encoder/*",
|
| 136 |
)],
|
| 137 |
}
|
| 138 |
model_dir = snapshot_download(model_id_or_path, **kw)
|
|
@@ -150,8 +150,8 @@ class Agent:
|
|
| 150 |
if not os.path.exists(cfg_path):
|
| 151 |
raise FileNotFoundError(
|
| 152 |
f"Incompatible model: {model_id_or_path!r} does not contain 'rl_agent_config.json'. "
|
| 153 |
-
f"That file ships with the weights of a
|
| 154 |
-
f"(e.g. 'doofz/
|
| 155 |
)
|
| 156 |
|
| 157 |
with open(cfg_path) as f:
|
|
@@ -195,7 +195,7 @@ class Agent:
|
|
| 195 |
self.model.load_state_dict(weights, strict=True)
|
| 196 |
|
| 197 |
# ModernBERT's reference_compile defaults to "auto" and will torch.compile the encoder.
|
| 198 |
-
# That is a loss for the batch sizes
|
| 199 |
# hang on some platforms, so keep the eager path.
|
| 200 |
try:
|
| 201 |
self.model.encoder.config.reference_compile = False
|
|
@@ -215,7 +215,7 @@ class Agent:
|
|
| 215 |
if clamp_temperature(t) != float(t)]
|
| 216 |
if rejected:
|
| 217 |
warnings.warn(
|
| 218 |
-
"
|
| 219 |
"confidence; clamping %s. Treat confidence from the affected buckets as uncalibrated."
|
| 220 |
% (TEMP_MIN, TEMP_MAX, ", ".join(rejected)),
|
| 221 |
RuntimeWarning, stacklevel=2)
|
|
@@ -247,7 +247,7 @@ class Agent:
|
|
| 247 |
|
| 248 |
if fell_back_from is not None:
|
| 249 |
print(
|
| 250 |
-
"\n[
|
| 251 |
" Reason: %s\n"
|
| 252 |
" Inference will be roughly 10-15x slower (~200-500 ms rather than ~35 ms).\n"
|
| 253 |
" If this is a newer NVIDIA GPU (Blackwell / RTX 50-series), your PyTorch build\n"
|
|
@@ -382,7 +382,7 @@ class Agent:
|
|
| 382 |
rest = [{k: v for k, v in qs.items() if (i, k) not in set(big)} for i, qs in enumerate(qlist)]
|
| 383 |
keep = [i for i, r in enumerate(rest) if r]
|
| 384 |
sub = self.predict_batch([states[i] for i in keep], [rest[i] for i in keep], batch_size, _tournament=False) if keep else []
|
| 385 |
-
res = [{"model": "
|
| 386 |
for i, r in zip(keep, sub):
|
| 387 |
res[i] = r
|
| 388 |
for (i, qid), a in zip(big, answers):
|
|
@@ -425,7 +425,7 @@ class Agent:
|
|
| 425 |
all_act[i, :ac.shape[1]] = ac[j]
|
| 426 |
out, off = [], 0
|
| 427 |
for qs, its in zip(qlist, per_state):
|
| 428 |
-
out.append({"model": "
|
| 429 |
"answers": self._decode(qs, its, all_logits, all_act, offset=off),
|
| 430 |
"usage": {"input_tokens": sum(len(it["ids"]) for it in its), "output_tokens": 0}})
|
| 431 |
off += len(its)
|
|
@@ -496,7 +496,7 @@ class Agent:
|
|
| 496 |
answers = self._decode(questions, items, logits, act)
|
| 497 |
|
| 498 |
return {
|
| 499 |
-
"model": "
|
| 500 |
"answers": answers,
|
| 501 |
"usage": {"input_tokens": n_tokens, "output_tokens": 0},
|
| 502 |
}
|
|
@@ -507,13 +507,13 @@ class Agent:
|
|
| 507 |
RLAgent = Agent
|
| 508 |
|
| 509 |
|
| 510 |
-
def load(model_id_or_path: str = "doofz/
|
| 511 |
token: Optional[str] = None, subfolder: Optional[str] = None, half: bool = True) -> Agent:
|
| 512 |
-
"""Load a
|
| 513 |
|
| 514 |
`subfolder` picks one checkpoint out of a repo that bundles several:
|
| 515 |
|
| 516 |
-
|
| 517 |
-
|
| 518 |
"""
|
| 519 |
return Agent(model_id_or_path, device=device, token=token, subfolder=subfolder, half=half)
|
|
|
|
| 1 |
+
"""High-level inference runtime for system01 System 1 decision models."""
|
| 2 |
import json
|
| 3 |
import os
|
| 4 |
import warnings
|
|
|
|
| 102 |
|
| 103 |
def __init__(
|
| 104 |
self,
|
| 105 |
+
model_id_or_path: str = "doofz/System01",
|
| 106 |
device: Optional[str] = None,
|
| 107 |
token: Optional[str] = None,
|
| 108 |
subfolder: Optional[str] = None,
|
| 109 |
half: bool = True,
|
| 110 |
):
|
| 111 |
+
"""Load a System01 checkpoint.
|
| 112 |
|
| 113 |
`subfolder` selects one checkpoint from a repo that bundles several, e.g.
|
| 114 |
+
`Agent("doofz/System01", subfolder="multilingual")`. Only that subfolder is
|
| 115 |
downloaded, so bundling does not cost every user the whole family.
|
| 116 |
"""
|
| 117 |
from safetensors.torch import load_file
|
|
|
|
| 132 |
kw = {
|
| 133 |
"token": token or os.environ.get("HF_TOKEN"),
|
| 134 |
"allow_patterns": [prefix + name for name in (
|
| 135 |
+
"config.json", "rl_agent_config.json", "model.safetensors", "tokenizer/*", "encoder/*",
|
| 136 |
)],
|
| 137 |
}
|
| 138 |
model_dir = snapshot_download(model_id_or_path, **kw)
|
|
|
|
| 150 |
if not os.path.exists(cfg_path):
|
| 151 |
raise FileNotFoundError(
|
| 152 |
f"Incompatible model: {model_id_or_path!r} does not contain 'rl_agent_config.json'. "
|
| 153 |
+
f"That file ships with the weights of a System01 checkpoint, so load one of those "
|
| 154 |
+
f"(e.g. 'doofz/System01') or a directory your own training run wrote."
|
| 155 |
)
|
| 156 |
|
| 157 |
with open(cfg_path) as f:
|
|
|
|
| 195 |
self.model.load_state_dict(weights, strict=True)
|
| 196 |
|
| 197 |
# ModernBERT's reference_compile defaults to "auto" and will torch.compile the encoder.
|
| 198 |
+
# That is a loss for the batch sizes System01 runs (a handful of questions per call) and can
|
| 199 |
# hang on some platforms, so keep the eager path.
|
| 200 |
try:
|
| 201 |
self.model.encoder.config.reference_compile = False
|
|
|
|
| 215 |
if clamp_temperature(t) != float(t)]
|
| 216 |
if rejected:
|
| 217 |
warnings.warn(
|
| 218 |
+
"system01: this checkpoint ships temperatures outside [%g, %g] which would distort "
|
| 219 |
"confidence; clamping %s. Treat confidence from the affected buckets as uncalibrated."
|
| 220 |
% (TEMP_MIN, TEMP_MAX, ", ".join(rejected)),
|
| 221 |
RuntimeWarning, stacklevel=2)
|
|
|
|
| 247 |
|
| 248 |
if fell_back_from is not None:
|
| 249 |
print(
|
| 250 |
+
"\n[system01] Warning: could not place the model on %s, so it is running on CPU.\n"
|
| 251 |
" Reason: %s\n"
|
| 252 |
" Inference will be roughly 10-15x slower (~200-500 ms rather than ~35 ms).\n"
|
| 253 |
" If this is a newer NVIDIA GPU (Blackwell / RTX 50-series), your PyTorch build\n"
|
|
|
|
| 382 |
rest = [{k: v for k, v in qs.items() if (i, k) not in set(big)} for i, qs in enumerate(qlist)]
|
| 383 |
keep = [i for i, r in enumerate(rest) if r]
|
| 384 |
sub = self.predict_batch([states[i] for i in keep], [rest[i] for i in keep], batch_size, _tournament=False) if keep else []
|
| 385 |
+
res = [{"model": "System01", "answers": {}, "usage": {"input_tokens": 0, "output_tokens": 0}} for _ in states]
|
| 386 |
for i, r in zip(keep, sub):
|
| 387 |
res[i] = r
|
| 388 |
for (i, qid), a in zip(big, answers):
|
|
|
|
| 425 |
all_act[i, :ac.shape[1]] = ac[j]
|
| 426 |
out, off = [], 0
|
| 427 |
for qs, its in zip(qlist, per_state):
|
| 428 |
+
out.append({"model": "System01",
|
| 429 |
"answers": self._decode(qs, its, all_logits, all_act, offset=off),
|
| 430 |
"usage": {"input_tokens": sum(len(it["ids"]) for it in its), "output_tokens": 0}})
|
| 431 |
off += len(its)
|
|
|
|
| 496 |
answers = self._decode(questions, items, logits, act)
|
| 497 |
|
| 498 |
return {
|
| 499 |
+
"model": "System01",
|
| 500 |
"answers": answers,
|
| 501 |
"usage": {"input_tokens": n_tokens, "output_tokens": 0},
|
| 502 |
}
|
|
|
|
| 507 |
RLAgent = Agent
|
| 508 |
|
| 509 |
|
| 510 |
+
def load(model_id_or_path: str = "doofz/System01", device: Optional[str] = None,
|
| 511 |
token: Optional[str] = None, subfolder: Optional[str] = None, half: bool = True) -> Agent:
|
| 512 |
+
"""Load a System01 agent.
|
| 513 |
|
| 514 |
`subfolder` picks one checkpoint out of a repo that bundles several:
|
| 515 |
|
| 516 |
+
system01.load("doofz/System01") # English (repo root)
|
| 517 |
+
system01.load("doofz/System01", subfolder="multilingual")
|
| 518 |
"""
|
| 519 |
return Agent(model_id_or_path, device=device, token=token, subfolder=subfolder, half=half)
|
{system1 → system01}/common.py
RENAMED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
"""Core model architecture, token sequence construction, and confidence estimation for
|
| 2 |
import json
|
| 3 |
import math
|
| 4 |
import os
|
|
|
|
| 1 |
+
"""Core model architecture, token sequence construction, and confidence estimation for system01."""
|
| 2 |
import json
|
| 3 |
import math
|
| 4 |
import os
|
{system1 → system01}/email.py
RENAMED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
"""Email utilities for cleaning and structuring email inputs in
|
| 2 |
import re
|
| 3 |
from typing import Dict, Optional
|
| 4 |
|
|
|
|
| 1 |
+
"""Email utilities for cleaning and structuring email inputs in system01."""
|
| 2 |
import re
|
| 3 |
from typing import Dict, Optional
|
| 4 |
|
{system1 → system01}/lang.py
RENAMED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
"""Dependency-free language/script detection used to route between
|
| 2 |
|
| 3 |
Routing only needs one decision: *is this English Latin text, or is it something the English
|
| 4 |
checkpoint cannot read?* Benchmarks on MASSIVE (14 languages) showed the English checkpoint
|
|
|
|
| 1 |
+
"""Dependency-free language/script detection used to route between System01 checkpoints.
|
| 2 |
|
| 3 |
Routing only needs one decision: *is this English Latin text, or is it something the English
|
| 4 |
checkpoint cannot read?* Benchmarks on MASSIVE (14 languages) showed the English checkpoint
|
{system1 → system01}/presets.py
RENAMED
|
File without changes
|
{system1 → system01}/server.py
RENAMED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
-
"""HTTP API for
|
| 2 |
|
| 3 |
-
|
| 4 |
|
| 5 |
Endpoints
|
| 6 |
GET /health liveness
|
|
@@ -22,14 +22,14 @@ from . import __version__, presets
|
|
| 22 |
from .agent import Agent
|
| 23 |
from .lang import analyse
|
| 24 |
|
| 25 |
-
MODEL = os.environ.get("
|
| 26 |
-
API_KEY = os.environ.get("
|
| 27 |
|
| 28 |
-
app = FastAPI(title="
|
| 29 |
-
description="
|
| 30 |
agent: Optional[Agent] = None
|
| 31 |
-
BATCH_WINDOW_MS = float(os.environ.get("
|
| 32 |
-
BATCH_MAX = int(os.environ.get("
|
| 33 |
_queue: Optional[asyncio.Queue] = None
|
| 34 |
|
| 35 |
|
|
@@ -84,19 +84,19 @@ class BatchRequest(BaseModel):
|
|
| 84 |
@app.on_event("startup")
|
| 85 |
async def _startup():
|
| 86 |
global agent, _queue
|
| 87 |
-
agent = Agent(MODEL, device=os.environ.get("
|
| 88 |
_queue = asyncio.Queue()
|
| 89 |
asyncio.get_running_loop().create_task(_batcher())
|
| 90 |
|
| 91 |
|
| 92 |
@app.get("/health")
|
| 93 |
def health():
|
| 94 |
-
return {"status": "ok", "model": "
|
| 95 |
|
| 96 |
|
| 97 |
@app.get("/v1/models", dependencies=[Depends(auth)])
|
| 98 |
def models():
|
| 99 |
-
return {"object": "list", "data": [{"id": "
|
| 100 |
|
| 101 |
|
| 102 |
@app.get("/v1/presets", dependencies=[Depends(auth)])
|
|
|
|
| 1 |
+
"""HTTP API for System01: calibrated typed decisions over REST.
|
| 2 |
|
| 3 |
+
SYSTEM01_MODEL=doofz/System01 SYSTEM01_API_KEY=... python -m system01.server --port 8095
|
| 4 |
|
| 5 |
Endpoints
|
| 6 |
GET /health liveness
|
|
|
|
| 22 |
from .agent import Agent
|
| 23 |
from .lang import analyse
|
| 24 |
|
| 25 |
+
MODEL = os.environ.get("SYSTEM01_MODEL", "doofz/System01")
|
| 26 |
+
API_KEY = os.environ.get("SYSTEM01_API_KEY")
|
| 27 |
|
| 28 |
+
app = FastAPI(title="System01 API", version=__version__,
|
| 29 |
+
description="System01: non-autoregressive, calibrated decisions in one forward pass.")
|
| 30 |
agent: Optional[Agent] = None
|
| 31 |
+
BATCH_WINDOW_MS = float(os.environ.get("SYSTEM01_BATCH_WINDOW_MS", "3"))
|
| 32 |
+
BATCH_MAX = int(os.environ.get("SYSTEM01_BATCH_MAX", "256"))
|
| 33 |
_queue: Optional[asyncio.Queue] = None
|
| 34 |
|
| 35 |
|
|
|
|
| 84 |
@app.on_event("startup")
|
| 85 |
async def _startup():
|
| 86 |
global agent, _queue
|
| 87 |
+
agent = Agent(MODEL, device=os.environ.get("SYSTEM01_DEVICE", "cuda"))
|
| 88 |
_queue = asyncio.Queue()
|
| 89 |
asyncio.get_running_loop().create_task(_batcher())
|
| 90 |
|
| 91 |
|
| 92 |
@app.get("/health")
|
| 93 |
def health():
|
| 94 |
+
return {"status": "ok", "model": "System01", "version": __version__}
|
| 95 |
|
| 96 |
|
| 97 |
@app.get("/v1/models", dependencies=[Depends(auth)])
|
| 98 |
def models():
|
| 99 |
+
return {"object": "list", "data": [{"id": "System01", "source": MODEL, "device": str(agent.device)}]}
|
| 100 |
|
| 101 |
|
| 102 |
@app.get("/v1/presets", dependencies=[Depends(auth)])
|
{system1 → system01}/shortlist.py
RENAMED
|
@@ -116,7 +116,7 @@ def embed_fn_from_agent(
|
|
| 116 |
The callable embeds a list of strings with ``agent.tok`` and ``agent.model.encoder``.
|
| 117 |
It does not run the decision head and does not download weights. A dedicated
|
| 118 |
bi-encoder passed as ``embed_fn`` will usually shortlist better; this helper is
|
| 119 |
-
for callers who only have the
|
| 120 |
|
| 121 |
Padding positions are excluded from the mean. The encoder's train/eval flag is
|
| 122 |
left as the caller set it (a loaded ``Agent`` is already in eval).
|
|
|
|
| 116 |
The callable embeds a list of strings with ``agent.tok`` and ``agent.model.encoder``.
|
| 117 |
It does not run the decision head and does not download weights. A dedicated
|
| 118 |
bi-encoder passed as ``embed_fn`` will usually shortlist better; this helper is
|
| 119 |
+
for callers who only have the System01 checkpoint in memory.
|
| 120 |
|
| 121 |
Padding positions are excluded from the mean. The encoder's train/eval flag is
|
| 122 |
left as the caller set it (a loaded ``Agent`` is already in eval).
|
train/soup.py
CHANGED
|
@@ -3,20 +3,22 @@ import json, shutil, sys, os
|
|
| 3 |
import numpy as np, torch
|
| 4 |
sys.path.insert(0, "./train")
|
| 5 |
import train_v2 as T
|
| 6 |
-
|
| 7 |
-
import
|
| 8 |
from safetensors.torch import load_file, save_file
|
| 9 |
|
| 10 |
alpha = float(sys.argv[1]); OUT = sys.argv[2]
|
| 11 |
-
|
|
|
|
|
|
|
| 12 |
os.makedirs(OUT, exist_ok=True)
|
| 13 |
save_file({k: ((1 - alpha) * a[k].float() + alpha * b[k].float()).to(torch.float16).contiguous() for k in a}, f"{OUT}/model.safetensors")
|
| 14 |
for sub in ("encoder", "tokenizer"):
|
| 15 |
-
shutil.copytree(f"
|
| 16 |
-
cfg = json.load(open("
|
| 17 |
cfg["temperature_by_options"] = {}; cfg["temperature"] = [1.0, 1.0, 1.0]
|
| 18 |
json.dump(cfg, open(f"{OUT}/rl_agent_config.json", "w"), indent=2)
|
| 19 |
-
ag =
|
| 20 |
val = T.synth_items("val")
|
| 21 |
for l in open(f"{T.SYN}/spatial_val.jsonl"):
|
| 22 |
r = json.loads(l)
|
|
|
|
| 3 |
import numpy as np, torch
|
| 4 |
sys.path.insert(0, "./train")
|
| 5 |
import train_v2 as T
|
| 6 |
+
sys.path.insert(0, ".") # repo root, so `import system01` works
|
| 7 |
+
import system01
|
| 8 |
from safetensors.torch import load_file, save_file
|
| 9 |
|
| 10 |
alpha = float(sys.argv[1]); OUT = sys.argv[2]
|
| 11 |
+
A_PATH = sys.argv[3] if len(sys.argv) > 3 else "."
|
| 12 |
+
B_PATH = sys.argv[4] if len(sys.argv) > 4 else "./ckpt/system1_v4"
|
| 13 |
+
a = load_file(f"{A_PATH}/model.safetensors"); b = load_file(f"{B_PATH}/model.safetensors")
|
| 14 |
os.makedirs(OUT, exist_ok=True)
|
| 15 |
save_file({k: ((1 - alpha) * a[k].float() + alpha * b[k].float()).to(torch.float16).contiguous() for k in a}, f"{OUT}/model.safetensors")
|
| 16 |
for sub in ("encoder", "tokenizer"):
|
| 17 |
+
shutil.copytree(f"{A_PATH}/{sub}", f"{OUT}/{sub}", dirs_exist_ok=True)
|
| 18 |
+
cfg = json.load(open(f"{B_PATH}/rl_agent_config.json"))
|
| 19 |
cfg["temperature_by_options"] = {}; cfg["temperature"] = [1.0, 1.0, 1.0]
|
| 20 |
json.dump(cfg, open(f"{OUT}/rl_agent_config.json", "w"), indent=2)
|
| 21 |
+
ag = system01.load(OUT, device="cuda")
|
| 22 |
val = T.synth_items("val")
|
| 23 |
for l in open(f"{T.SYN}/spatial_val.jsonl"):
|
| 24 |
r = json.loads(l)
|
train/train_v2.py
CHANGED
|
@@ -14,8 +14,8 @@ import hashlib, json, math, os, random, shutil, sys, time
|
|
| 14 |
import numpy as np, torch
|
| 15 |
ROOT = "."
|
| 16 |
sys.path.insert(0, f"{ROOT}/pkg"); sys.path.insert(0, f"{ROOT}/bench")
|
| 17 |
-
import
|
| 18 |
-
from
|
| 19 |
from safetensors.torch import save_file
|
| 20 |
import bitsandbytes as bnb
|
| 21 |
|
|
@@ -205,7 +205,7 @@ def main():
|
|
| 205 |
for sub in ("encoder", "tokenizer"):
|
| 206 |
shutil.copytree(os.path.join(os.environ.get("BASE_CKPT", "base"), sub), f"{OUT}/{sub}", dirs_exist_ok=True)
|
| 207 |
cfg = dict(ag.cfg)
|
| 208 |
-
cfg.update({"model_name": "
|
| 209 |
"training": {"objective": "RLCD strictly proper scoring reward (log + spherical + RPS)",
|
| 210 |
"steps": steps, "epochs": EPOCHS, "items": len(data),
|
| 211 |
"minutes": round((time.time() - t0) / 60, 1),
|
|
|
|
| 14 |
import numpy as np, torch
|
| 15 |
ROOT = "."
|
| 16 |
sys.path.insert(0, f"{ROOT}/pkg"); sys.path.insert(0, f"{ROOT}/bench")
|
| 17 |
+
import system01 as systemone, tasks
|
| 18 |
+
from system01.common import QTYPES, build_sequence, collate_items, proper_reward, render_options, temp_bucket
|
| 19 |
from safetensors.torch import save_file
|
| 20 |
import bitsandbytes as bnb
|
| 21 |
|
|
|
|
| 205 |
for sub in ("encoder", "tokenizer"):
|
| 206 |
shutil.copytree(os.path.join(os.environ.get("BASE_CKPT", "base"), sub), f"{OUT}/{sub}", dirs_exist_ok=True)
|
| 207 |
cfg = dict(ag.cfg)
|
| 208 |
+
cfg.update({"model_name": "System01", "temperature_by_options": temps, "auto_head_budget": True,
|
| 209 |
"training": {"objective": "RLCD strictly proper scoring reward (log + spherical + RPS)",
|
| 210 |
"steps": steps, "epochs": EPOCHS, "items": len(data),
|
| 211 |
"minutes": round((time.time() - t0) / 60, 1),
|
train/train_v4.py
CHANGED
|
@@ -1,21 +1,21 @@
|
|
| 1 |
-
"""Continue
|
| 2 |
import json, os, random, shutil, sys, time
|
| 3 |
import numpy as np, torch
|
| 4 |
sys.path.insert(0, "./train")
|
| 5 |
import train_v2 as T
|
| 6 |
from safetensors.torch import save_file
|
| 7 |
import bitsandbytes as bnb
|
| 8 |
-
|
| 9 |
|
| 10 |
ROOT = T.ROOT
|
| 11 |
-
INIT = f"{ROOT}/release2"
|
| 12 |
-
OUT = f"{ROOT}/ckpt/system1_v4"
|
| 13 |
-
T.CURVE = f"{ROOT}/results/train_curve_v4.jsonl"
|
| 14 |
rng = random.Random(1)
|
| 15 |
|
| 16 |
|
| 17 |
def main():
|
| 18 |
-
import
|
| 19 |
# split spatial by group
|
| 20 |
rows = [json.loads(l) for l in open(f"{T.SYN}/spatial.jsonl")]
|
| 21 |
fh = {s: open(f"{T.SYN}/spatial_{s}.jsonl", "w") for s in ("train", "val", "test")}
|
|
@@ -30,10 +30,16 @@ def main():
|
|
| 30 |
sp.append((r["state"], {qid: r["questions"][qid]}, {qid: g}))
|
| 31 |
replay = T.build_train()
|
| 32 |
rng.shuffle(replay)
|
| 33 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
rng.shuffle(mix)
|
| 35 |
|
| 36 |
-
ag =
|
| 37 |
model = ag.model; model.train()
|
| 38 |
emb = model.encoder.embeddings.tok_embeddings
|
| 39 |
emb.weight.requires_grad_(False); emb.to(torch.bfloat16)
|
|
|
|
| 1 |
+
"""Continue System01 (v3) on procedurally generated spatial-control decisions + replay of the v3 mix."""
|
| 2 |
import json, os, random, shutil, sys, time
|
| 3 |
import numpy as np, torch
|
| 4 |
sys.path.insert(0, "./train")
|
| 5 |
import train_v2 as T
|
| 6 |
from safetensors.torch import save_file
|
| 7 |
import bitsandbytes as bnb
|
| 8 |
+
sys.path.insert(0, ".") # repo root, so `import system01` works
|
| 9 |
|
| 10 |
ROOT = T.ROOT
|
| 11 |
+
INIT = os.environ.get("INIT", f"{ROOT}/release2")
|
| 12 |
+
OUT = os.environ.get("OUT", f"{ROOT}/ckpt/system1_v4")
|
| 13 |
+
T.CURVE = os.environ.get("CURVE", f"{ROOT}/results/train_curve_v4.jsonl")
|
| 14 |
rng = random.Random(1)
|
| 15 |
|
| 16 |
|
| 17 |
def main():
|
| 18 |
+
import system01
|
| 19 |
# split spatial by group
|
| 20 |
rows = [json.loads(l) for l in open(f"{T.SYN}/spatial.jsonl")]
|
| 21 |
fh = {s: open(f"{T.SYN}/spatial_{s}.jsonl", "w") for s in ("train", "val", "test")}
|
|
|
|
| 30 |
sp.append((r["state"], {qid: r["questions"][qid]}, {qid: g}))
|
| 31 |
replay = T.build_train()
|
| 32 |
rng.shuffle(replay)
|
| 33 |
+
if os.environ.get("STAGE") == "routing":
|
| 34 |
+
# stage 3: LLM/agent routing emphasis (short queries), spatial kept alive, general replay
|
| 35 |
+
route = [x for x in T.synth_items("train") if "route" in x[1]]
|
| 36 |
+
rng.shuffle(route)
|
| 37 |
+
mix = route[:14000] + sp[:4000] + replay[:30000]
|
| 38 |
+
else:
|
| 39 |
+
mix = sp + replay[:45000]
|
| 40 |
rng.shuffle(mix)
|
| 41 |
|
| 42 |
+
ag = system01.load(INIT, device="cuda", half=False)
|
| 43 |
model = ag.model; model.train()
|
| 44 |
emb = model.encoder.embeddings.tok_embeddings
|
| 45 |
emb.weight.requires_grad_(False); emb.to(torch.bfloat16)
|