feat: add compiled serving shared training and benchmarks
Browse files- .gitattributes +1 -36
- .gitignore +11 -3
- BENCHMARKS.md +114 -0
- CHANGES.md +4 -0
- MODEL_CARD.md +170 -0
- README.md +16 -5
- RUNNING.md +13 -1
- TRAINING.md +11 -0
- evaluation/benchmarks/fev/predictions.jsonl +0 -0
- evaluation/benchmarks/fev/report.json +675 -0
- evaluation/benchmarks/fev/rows.json +0 -0
- evaluation/benchmarks/jev/predictions.jsonl +0 -0
- evaluation/benchmarks/jev/report.json +678 -0
- evaluation/benchmarks/jev/rows.json +0 -0
- evaluation/benchmarks/kev/predictions.jsonl +0 -0
- evaluation/benchmarks/kev/report.json +675 -0
- evaluation/benchmarks/kev/rows.json +0 -0
- evaluation/benchmarks/results.json +2334 -0
- evaluation/benchmarks/runtime.json +60 -0
- evaluation/benchmarks/russian-typed.jsonl +48 -0
- evaluation/benchmarks/training-validation.json +218 -0
- examples/training.jsonl +40 -0
- fev4b/adaptation.py +51 -0
- fev4b/benchmark.py +97 -0
- fev4b/calibrate.py +68 -0
- fev4b/data.py +140 -0
- fev4b/integrity.py +1 -1
- fev4b/metrics.py +74 -0
- fev4b/model.py +18 -2
- fev4b/train.py +150 -0
- fev4b/version.py +1 -0
- fev_config.json +1 -1
- manifest.json +183 -59
- provenance.json +15 -2
- run.py +4 -1
- serve.py +4 -1
- skills/fev-finetune/SKILL.md +21 -0
- skills/fev-finetune/references/training.md +98 -0
- tests/test_training_tools.py +107 -0
- tools/benchmark_kev.py +63 -0
- tools/compare_runs.py +82 -0
- tools/prepare_benchmark.py +46 -0
- tools/train_hf.py +59 -0
.gitattributes
CHANGED
|
@@ -1,36 +1 @@
|
|
| 1 |
-
|
| 2 |
-
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
-
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
-
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 1 |
+
LICENSE -text whitespace=cr-at-eol
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.gitignore
CHANGED
|
@@ -1,6 +1,14 @@
|
|
| 1 |
.venv/
|
| 2 |
__pycache__/
|
| 3 |
-
*.
|
| 4 |
-
.cache/
|
| 5 |
.env
|
| 6 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
.venv/
|
| 2 |
__pycache__/
|
| 3 |
+
*.py[cod]
|
|
|
|
| 4 |
.env
|
| 5 |
+
.env.*
|
| 6 |
+
models/
|
| 7 |
+
checkpoint/
|
| 8 |
+
adapter/
|
| 9 |
+
*.safetensors
|
| 10 |
+
*.pt
|
| 11 |
+
*.bin
|
| 12 |
+
.DS_Store
|
| 13 |
+
work/
|
| 14 |
+
cloud-inputs/
|
BENCHMARKS.md
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Fev benchmarks
|
| 2 |
+
|
| 3 |
+
## Native typed-decision comparison
|
| 4 |
+
|
| 5 |
+
This comparison sends exactly the same 812 requests (908 questions) to frozen Fev-4B, Kev-4B and Jev-1.14.0. It measures the native typed interfaces, without Fev's two-stage conversation-routing policy. It does not compare generated answers or completed conversations.
|
| 6 |
+
|
| 7 |
+
- 764 public records from Kev's `transfer-v4` **development** split: 444 Choice, 280 Noul and 40 Score questions. States, instructions, criteria, order and labels are preserved. The suite spans public knowledge, routing, entailment, sentiment and ordinal tasks. The original source metadata and licenses remain with their respective datasets.
|
| 8 |
+
- 48 authored Russian records with 144 questions: Choice, Noul and Score on explicit fictional facts across six domains. They repeat eight templates; this is a diagnostic, not independent customer accuracy. These inputs were not used by the new trainer.
|
| 9 |
+
- No private customer data appears in the published benchmark artifacts. No upstream-pretraining disjointness claim is made. Results from an already public development suite can inform engineering comparisons but cannot establish an unbiased unseen-task leaderboard.
|
| 10 |
+
|
| 11 |
+
Inputs SHA256: `9266d6ea1ce83c9755a60cf48c7469b0fa3264c229e96f4142d683bada8fef91`.
|
| 12 |
+
|
| 13 |
+
| System | Revision and execution |
|
| 14 |
+
|---|---|
|
| 15 |
+
| Fev-4B | Hub `3ef077cd7cc3c97d696c18150358ab4de3a6884e`; BF16 backbone, FP32 shared adapter unmerged; reference native readout |
|
| 16 |
+
| Kev-4B | Hub `6cfce5c2fa4b4bd64026336ab649c5ca78857d52`; official code `2074cc44256200bb37540103fa4572901faa10f3`; BF16 merged, eager SDPA, no fused path/graphs |
|
| 17 |
+
| Jev | Hosted API pinned to `jev-1.14.0`; served model recorded per response; server hardware and precision undisclosed |
|
| 18 |
+
|
| 19 |
+
The two open models use the same A10G GPU class in separate jobs and their own supported runtime/dependency versions. This is a deployment-precision comparison, not Kev's published FP32 evaluation. No temperature or threshold is refitted on these inputs. Fev processes questions jointly; Kev isolates question rows. Those architectural differences remain part of the measured behavior.
|
| 20 |
+
|
| 21 |
+
Top-option accuracy counts failures as incorrect. Noul is thresholded at 0.5; Score uses the most probable rubric level, with ordinal error reported separately. Brier/NLL/ECE and fixed-threshold coverage use top-option probability, not the API's normalized `confidence` field. Distributions are normalized only when their mass is within 0.02 of one, allowing hosted probability rounding; raw responses remain preserved. A small-sample ECE does not establish calibrated probabilities on new workloads.
|
| 22 |
+
|
| 23 |
+
### Results (2026-10-24)
|
| 24 |
+
|
| 25 |
+
| Measure | Fev-4B | Kev-4B | Jev-1.14.0 |
|
| 26 |
+
|---|---:|---:|---:|
|
| 27 |
+
| Public development (764 questions) | 639/764 (83.6%) | 613/764 (80.2%) | 646/764 (84.6%) |
|
| 28 |
+
| Authored Russian (144 questions) | 144/144 (100.0%) | 143/144 (99.3%) | 144/144 (100.0%) |
|
| 29 |
+
| Combined (908 questions) | 783/908 (86.2%) | 756/908 (83.3%) | 790/908 (87.0%) |
|
| 30 |
+
| Brier, combined (lower is better) | 0.1962 | 0.2305 | 0.1893 |
|
| 31 |
+
| NLL, combined (lower is better) | 0.3740 | 0.4291 | 0.8839 |
|
| 32 |
+
| ECE10, combined (lower is better) | 0.0421 | 0.0516 | 0.0466 |
|
| 33 |
+
| Confident wrong, probability ≥0.9 | 26 | 6 | 36 |
|
| 34 |
+
| Failed responses | 0 | 0 | 0 |
|
| 35 |
+
|
| 36 |
+
Fev is 7/908 answers behind Jev and 27/908 ahead of Kev on this particular combined set. The combined score weights each question equally and includes the easier repetitive Russian probes; the public-only row is the more useful broad comparison. This is not evidence of a statistically significant ranking or an unseen-customer success rate. NLL clips zero probabilities at 1e-12; the hosted API’s rounded zeros affect that metric.
|
| 37 |
+
|
| 38 |
+
| Public source | Questions | Fev | Kev | Jev |
|
| 39 |
+
|---|---:|---:|---:|---:|
|
| 40 |
+
| composition_holdout | 96 | 87.5% | 97.9% | 89.6% |
|
| 41 |
+
| emotion | 116 | 59.5% | 56.0% | 60.3% |
|
| 42 |
+
| legacy_holdout | 80 | 96.2% | 81.2% | 98.8% |
|
| 43 |
+
| mmlu | 116 | 74.1% | 68.1% | 83.6% |
|
| 44 |
+
| paws | 80 | 85.0% | 80.0% | 78.8% |
|
| 45 |
+
| qnli | 80 | 92.5% | 91.2% | 91.2% |
|
| 46 |
+
| sciq | 116 | 97.4% | 98.3% | 97.4% |
|
| 47 |
+
| tweet_offensive | 80 | 85.0% | 73.8% | 81.2% |
|
| 48 |
+
|
| 49 |
+
| Typed question, combined | Questions | Fev | Kev | Jev |
|
| 50 |
+
|---|---:|---:|---:|---:|
|
| 51 |
+
| choice | 492 | 81.3% | 81.3% | 84.1% |
|
| 52 |
+
| noul | 328 | 90.9% | 86.6% | 88.1% |
|
| 53 |
+
| score | 88 | 96.6% | 81.8% | 98.9% |
|
| 54 |
+
|
| 55 |
+
Expected-score mean absolute error on the 40 public ordinal questions: Fev 0.1062, Kev 0.3423, Jev 0.0708. This complements the Score top-option metric.
|
| 56 |
+
|
| 57 |
+
| Fixed threshold ≥0.95, combined | Fev | Kev | Jev |
|
| 58 |
+
|---|---:|---:|---:|
|
| 59 |
+
| Observed coverage | 62.4% | 30.4% | 69.3% |
|
| 60 |
+
| Observed error among accepted questions | 1.2% | 0.7% | 4.3% |
|
| 61 |
+
|
| 62 |
+
This fixed threshold was not chosen using these results. Coverage is descriptive; it is not a validated future error budget or a recommendation to change routing policy. See [machine-readable results](evaluation/benchmarks/results.json), including all six fixed thresholds and paired outcomes. Raw public/authored predictions are included beside it.
|
| 63 |
+
|
| 64 |
+
### Reproduce
|
| 65 |
+
|
| 66 |
+
```bash
|
| 67 |
+
.venv/bin/python tools/prepare_benchmark.py --out work/common-inputs
|
| 68 |
+
.venv/bin/python -m fev4b.benchmark --model-dir models/fev-4b \
|
| 69 |
+
--data work/common-inputs/comparison.jsonl --out work/fev-common
|
| 70 |
+
# Pinned Kev code/dependencies and BF16 model, on a CUDA server:
|
| 71 |
+
uv run tools/benchmark_kev.py --data work/common-inputs/comparison.jsonl --out work/kev-common
|
| 72 |
+
# Pinned hosted Jev, public/authored inputs only; set TYPESAFE_API_KEY in your shell:
|
| 73 |
+
.venv/bin/python -m fev4b.benchmark --endpoint https://api.typesafe.ai \
|
| 74 |
+
--model jev-1.14.0 --api-key-env TYPESAFE_API_KEY \
|
| 75 |
+
--data work/common-inputs/comparison.jsonl --out work/jev-common
|
| 76 |
+
# Against a System One server:
|
| 77 |
+
.venv/bin/python -m fev4b.benchmark --endpoint http://127.0.0.1:8000 \
|
| 78 |
+
--model fev-4b --data work/common-inputs/comparison.jsonl --out work/http-common
|
| 79 |
+
```
|
| 80 |
+
|
| 81 |
+
Use `--api-key-env NAME` to read credentials from an environment variable; do not put keys in datasets or committed commands. Remote use sends the selected dataset to that endpoint. The data-preparation script downloads only the pinned development file and checks its original and converted hashes. No test split is opened. Preserve predictions and failures rather than rerunning until a favorable result appears. Summarize saved runs without requerying:
|
| 82 |
+
|
| 83 |
+
```bash
|
| 84 |
+
.venv/bin/python -m tools.compare_runs --run Fev=work/fev-common \
|
| 85 |
+
--run Kev=work/kev-common --run Jev=work/jev-common --out work/comparison.json
|
| 86 |
+
```
|
| 87 |
+
|
| 88 |
+
## Conversation routing development evidence
|
| 89 |
+
|
| 90 |
+
The earlier routing composition had 293/303 reference agreement (96.7%) on consumed synthetic/source-derived development cases: 258 correct automatic routes, four automatic disagreements, 35 correct clarifications and six unnecessary clarifications. Five references had unresolved business mappings. The complete denominator is retained. These numbers are not native typed-task accuracy, production resolution, an independently labelled customer test or comparable to dashboard effectiveness percentages.
|
| 91 |
+
|
| 92 |
+
## Compiled runtime
|
| 93 |
+
|
| 94 |
+
The completed earlier regression preserved 303/303 final routing decisions. Internal candidate sets changed on 33 cases, so this is not numerical parity. Three selected routing workloads, five warm repeats each, measured on the same A10G and unchanged BF16/FP32 weights:
|
| 95 |
+
|
| 96 |
+
| Metric | Reference | Compiled |
|
| 97 |
+
|---|---:|---:|
|
| 98 |
+
| Warm median | 2,451.97 ms | 1,690.33 ms |
|
| 99 |
+
| Warm p95 | 2,539.22 ms | 1,733.43 ms |
|
| 100 |
+
| Peak allocated VRAM (decimal GB) | 9.997 | 10.086 |
|
| 101 |
+
|
| 102 |
+
This is a 31.1% warm median reduction and a 0.9% allocated-memory increase on those workloads. Compilation/load time, HTTP, queueing and concurrent throughput are excluded. It does not predict an H100 benchmark or general production speed. Reference remains the default; CUDA graphs, FP8, pruning and prefetching are disabled.
|
| 103 |
+
|
| 104 |
+
The packaged implementation additionally passed 6/6 routing controls, all 12 typed winners across four Russian requests, and authenticated TypeSafe SDK calls with plain JSON and zstd. The maximum typed probability change was 0.0016202. See [runtime evidence](evaluation/benchmarks/runtime.json).
|
| 105 |
+
|
| 106 |
+
## Training validation
|
| 107 |
+
|
| 108 |
+
The new trainer is checked on a separate, authored four-domain toy corpus. Its purpose is to establish training mechanics and adapter serialization, not improved accuracy. Train/development/calibration/test partitions are frozen before training; the public comparison and Russian diagnostic inputs never enter this trainer. No resulting toy adapter is adopted into Fev's released weights.
|
| 109 |
+
|
| 110 |
+
The completed smoke run took six steps on 28 records spanning four fictional domains. All 240 shared-adapter tensors changed (10,387,456 FP32 parameters), with finite gradients and the backbone/readout frozen. Development stayed 12/12 correct before and after; no accuracy improvement is established.
|
| 111 |
+
|
| 112 |
+
The first post-training reload assertion failed: maximum probability difference 0.00048164 against a 0.00001 tolerance, after compiled and HTTP work in that process. Its cause was not isolated. A fresh-process follow-up then verified all 240 saved tensors bit-for-bit, zero probability difference across repeated calls and against the saved post-training predictions, and all 12 option winners unchanged. Both outcomes are retained.
|
| 113 |
+
|
| 114 |
+
Separate calibration and final toy evaluation completed with 12/12 valid, correct test predictions. The temperature fit selected its lower search bound on this tiny all-correct sample; it is a machinery check, not useful evidence of production calibration. Toy templates recur across partitions, so this is not cross-domain generalization evidence. The toy adapter and fitted temperature are not adopted or distributed. See [training validation evidence](evaluation/benchmarks/training-validation.json).
|
CHANGES.md
CHANGED
|
@@ -1,5 +1,9 @@
|
|
| 1 |
# Release composition
|
| 2 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
## v0.2.0
|
| 4 |
|
| 5 |
Adds the text-only `Fev4B.decide` interface, a resident reference HTTP server for `POST /v1/systemone` and `GET /v1/models`, strict typed-request/response validation, zstd decoding, TypeSafe SDK contract checks, and public GitHub documentation. Choice, Noul and Score use the existing native decision head. No checkpoint, shared adapter, tokenizer, native source, temperature, precision, routing primitive or routing cutoff changed. The routing response version is now 0.2.0.
|
|
|
|
| 1 |
# Release composition
|
| 2 |
|
| 3 |
+
## v0.2.1
|
| 4 |
+
|
| 5 |
+
Adds opt-in compiled execution, an explicit verified adapter overlay, native shared-adapter training, grouped dataset splitting, separate temperature calibration, typed benchmark tooling and the `fev-finetune` agent skill. The baseline checkpoint, original shared adapter, tokenizer, native source, temperature and routing policy remain unchanged. Reference execution stays the default. Research training outputs are separate overlays and are not adopted or distributed as released weights. Benchmark methods and limitations are recorded in [BENCHMARKS.md](BENCHMARKS.md).
|
| 6 |
+
|
| 7 |
## v0.2.0
|
| 8 |
|
| 9 |
Adds the text-only `Fev4B.decide` interface, a resident reference HTTP server for `POST /v1/systemone` and `GET /v1/models`, strict typed-request/response validation, zstd decoding, TypeSafe SDK contract checks, and public GitHub documentation. Choice, Noul and Score use the existing native decision head. No checkpoint, shared adapter, tokenizer, native source, temperature, precision, routing primitive or routing cutoff changed. The routing response version is now 0.2.0.
|
MODEL_CARD.md
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
language:
|
| 3 |
+
- ru
|
| 4 |
+
license: apache-2.0
|
| 5 |
+
pipeline_tag: text-classification
|
| 6 |
+
tags:
|
| 7 |
+
- fev
|
| 8 |
+
- decision-model
|
| 9 |
+
- routing
|
| 10 |
+
- qwen3.5
|
| 11 |
+
- lora
|
| 12 |
+
- custom-code
|
| 13 |
+
---
|
| 14 |
+
|
| 15 |
+
# Fev-4B
|
| 16 |
+
|
| 17 |
+
## Model summary
|
| 18 |
+
|
| 19 |
+
Fev-4B is a decision model for classification, routing, triage and checks against supplied rules. Give it a document or conversation (the *state*) and named, typed questions. It returns option probabilities and structured decisions without generating text. Its text-decision server implements `POST /v1/systemone`, for use with the TypeSafe SDK through a custom base URL.
|
| 20 |
+
|
| 21 |
+
Built on the Qwen3.5-4B architecture, Fev combines a native decision readout with a shared LoRA adapter. Categories, business rules, examples and conversation history are supplied at runtime: customer-specific retraining is not required to change them. The package also includes a two-stage conversation router that can return a clarification signal when no route is sufficiently supported. Reference execution uses one 24 GB NVIDIA GPU.
|
| 22 |
+
|
| 23 |
+
## Model details
|
| 24 |
+
|
| 25 |
+
| | |
|
| 26 |
+
|---|---|
|
| 27 |
+
| Developer | Murad ([github.com/murabcd/fev](https://github.com/murabcd/fev)) |
|
| 28 |
+
| Model type | Decision model: shared state and questions scored by a native readout; no text generation |
|
| 29 |
+
| Backbone | Qwen3.5-4B architecture; noncausal full attention |
|
| 30 |
+
| Adapter | Shared LoRA, rank 8, alpha 16; 10,387,456 parameters |
|
| 31 |
+
| Precision | BF16 checkpoint and readout; FP32 adapter, unmerged |
|
| 32 |
+
| Head | Native readout over question option codes; softmax probabilities |
|
| 33 |
+
| Context | 16,384 encoded tokens per native call, including state, questions and template; oversized requests rejected |
|
| 34 |
+
| Temperature | Fixed native temperature T = 1.5539092221462472; new-workload calibration has not been measured |
|
| 35 |
+
| Languages | Russian |
|
| 36 |
+
| Reference hardware | NVIDIA A10G, 24 GB; CUDA BF16 |
|
| 37 |
+
| License | Apache-2.0; bundled notices retained |
|
| 38 |
+
| Version | Fev v0.2.1 |
|
| 39 |
+
|
| 40 |
+
**Input.** Text, a JSON object or an array as the state, plus 1–64 named questions:
|
| 41 |
+
|
| 42 |
+
| Type | Options | Output |
|
| 43 |
+
|---|---|---|
|
| 44 |
+
| `choice` | 1–255 named options with optional descriptions | Probability per option, selected option and confidence |
|
| 45 |
+
| `score` | 2–255 ordered level descriptions | Probability per level, expected zero-based level index, legend and confidence |
|
| 46 |
+
| `noul` | Yes / no, with optional descriptions | Probability of yes |
|
| 47 |
+
|
| 48 |
+
A typed request is processed as one joint sequence in one forward pass. Questions share attention; they are not isolated from one another. The context limit is an execution bound, not a validated accuracy guarantee for long documents.
|
| 49 |
+
|
| 50 |
+
## Intended uses
|
| 51 |
+
|
| 52 |
+
- Classify or route messages using supplied category definitions and exclusions.
|
| 53 |
+
- Make yes/no checks against supplied requirements.
|
| 54 |
+
- Rate a state against an explicit ordered rubric.
|
| 55 |
+
- Keep business configuration outside model weights.
|
| 56 |
+
- Run conversation routing with explicit uncertainty handling.
|
| 57 |
+
|
| 58 |
+
The model returns decisions. The application owns follow-up questions, scenario execution and business actions. A clarification signal can continue automation; it does not require operator handoff.
|
| 59 |
+
|
| 60 |
+
## Out-of-scope uses
|
| 61 |
+
|
| 62 |
+
- Chat, summarisation, free-form answers or other text generation.
|
| 63 |
+
- Image assets, box questions and multipart requests through this API.
|
| 64 |
+
- Claims of production accuracy, calibrated probabilities on arbitrary workloads, or autonomous conversation resolution based only on this development benchmark.
|
| 65 |
+
- CPU, Apple Silicon, quantized or merged-adapter serving: these paths are not validated in this release.
|
| 66 |
+
|
| 67 |
+
## How to use
|
| 68 |
+
|
| 69 |
+
Download the complete versioned model package. On a Linux server with a compatible NVIDIA GPU and driver:
|
| 70 |
+
|
| 71 |
+
```bash
|
| 72 |
+
uvx --from huggingface-hub==1.31.0 hf download murabcd/fev-4b \
|
| 73 |
+
--revision v0.2.1 --local-dir fev-4b
|
| 74 |
+
cd fev-4b
|
| 75 |
+
uv venv --python 3.12
|
| 76 |
+
uv pip install -r requirements.txt -r requirements-serve.txt
|
| 77 |
+
.venv/bin/python run.py --verify-only
|
| 78 |
+
.venv/bin/python serve.py --port 8000
|
| 79 |
+
```
|
| 80 |
+
|
| 81 |
+
The server keeps one model resident. Its default address is localhost. Code, examples and integration instructions are also on [GitHub](https://github.com/murabcd/fev).
|
| 82 |
+
|
| 83 |
+
```python
|
| 84 |
+
from typesafe_sdk import Choice, Noul, TypeSafeClient
|
| 85 |
+
|
| 86 |
+
client = TypeSafeClient(
|
| 87 |
+
api_key="local", base_url="http://127.0.0.1:8000",
|
| 88 |
+
model="fev-4b", timeout=120,
|
| 89 |
+
)
|
| 90 |
+
response = client.system_one(
|
| 91 |
+
state="Мне дважды списали оплату. Верните лишнее списание.",
|
| 92 |
+
questions={
|
| 93 |
+
"team": Choice(
|
| 94 |
+
instructions="Кто должен обработать обращение?",
|
| 95 |
+
criteria={"billing": "Оплата и возврат денег", "delivery": "Доставка"},
|
| 96 |
+
),
|
| 97 |
+
"refund": Noul(instructions="Клиент просит вернуть деньги?"),
|
| 98 |
+
},
|
| 99 |
+
)
|
| 100 |
+
print(response.choices["team"].choice, response.nouls["refund"].noul)
|
| 101 |
+
```
|
| 102 |
+
|
| 103 |
+
Install the client with `pip install 'typesafe-sdk[zstd]==0.7.5'`. Text requests support plain JSON and zstd compression. The server exposes `GET /v1/models` and `POST /v1/systemone`. See [API.md](API.md) for limits, authentication, raw HTTP and Python usage.
|
| 104 |
+
|
| 105 |
+
For the conversation-routing policy, use `Fev4B.route(...)` or `run.py --input examples/request.json`; see [RUNNING.md](RUNNING.md). This policy makes a primary Choice call and, when applicable, a second call containing up to three route-applicability checks. It is distinct from a single native typed request.
|
| 106 |
+
|
| 107 |
+
## Training and release procedure
|
| 108 |
+
|
| 109 |
+
This release distributes a complete decision checkpoint, one shared adapter and inference code. The shared adapter comes from customer-agnostic synthetic decision research. No customer-specific adapter is selected at runtime. The original checkpoint and shared adapter remain the baseline; new training tools produce explicit research overlays rather than overwriting them. See [the training guide](TRAINING.md) and the installable `fev-finetune` skill for native decision cross-entropy, grouped data splits and separate temperature calibration. A toy smoke run verifies mechanics; it is not a new accuracy claim or an adopted checkpoint.
|
| 110 |
+
|
| 111 |
+
Checkpoint and adapter files retain their original precision and hashes. Exact ancestry, adapter hashes and changed-file boundaries are recorded in [provenance.json](provenance.json) and [CHANGES.md](CHANGES.md). This is an inference release, not a complete reproduction of historical training data or an independent audit of all upstream sources.
|
| 112 |
+
|
| 113 |
+
## Evaluation
|
| 114 |
+
|
| 115 |
+
The new common native typed-decision comparison against pinned Kev-4B and Jev is documented in [BENCHMARKS.md](BENCHMARKS.md), with separate public-development and authored-Russian denominators, probability metrics and reproducibility commands. It is distinct from the routing-composition evidence below.
|
| 116 |
+
|
| 117 |
+
| Native comparison | Fev-4B | Kev-4B | Jev-1.14.0 |
|
| 118 |
+
|---|---:|---:|---:|
|
| 119 |
+
| Public development, 764 questions | 83.6% | 80.2% | 84.6% |
|
| 120 |
+
| Authored Russian, 144 questions | 100.0% | 99.3% | 100.0% |
|
| 121 |
+
|
| 122 |
+
All three systems received the same inputs, with zero failed responses. Open-model measurements use their BF16 serving paths; Jev uses its pinned hosted API. The public suite is already published development data, and Russian probes repeat templates. These are engineering comparisons, not unseen-customer accuracy. No comparison inputs were used by the new trainer.
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
The conversation-routing composition was measured on **303 development cases** with the full supplied business rules. Cases include synthetic and source-derived examples already consumed during development; this is not a fresh, independently human-labelled test set.
|
| 126 |
+
|
| 127 |
+
| Outcome | Cases |
|
| 128 |
+
|---|---:|
|
| 129 |
+
| Correct automatic route | 258 |
|
| 130 |
+
| Automatic route disagreement | 4 |
|
| 131 |
+
| Correct clarification | 35 |
|
| 132 |
+
| Unnecessary clarification | 6 |
|
| 133 |
+
| **Reference agreement** | **293 / 303 (96.7%)** |
|
| 134 |
+
|
| 135 |
+
Five references have unresolved business mappings, including all four automatic disagreements. Excluding those five gives 293 / 298 (98.3%). Both denominators are disclosed; the headline keeps the disputed cases.
|
| 136 |
+
|
| 137 |
+
| Development group | Agreement |
|
| 138 |
+
|---|---:|
|
| 139 |
+
| Earlier routing checks | 62 / 64 |
|
| 140 |
+
| Taxonomy checks | 54 / 55 |
|
| 141 |
+
| Boundary and transfer checks | 47 / 52 |
|
| 142 |
+
| Generic decision checks | 98 / 100 |
|
| 143 |
+
| Runtime instruction switches | 32 / 32 |
|
| 144 |
+
|
| 145 |
+
There were 48 / 48 consistent route-order pairs and 16 / 16 instruction-switch pairs with both answers correct. Eight same-job native repeat checks had no numerical change. [Aggregate results](evaluation/summary.json) are included; customer records and configurations are excluded.
|
| 146 |
+
|
| 147 |
+
These results measure the routing composition. They do not establish general typed-task accuracy, unseen-customer generalization, resolved-conversation coverage or production readiness. API contract tests are separate from model-quality evaluation.
|
| 148 |
+
|
| 149 |
+
**Release verification.** The complete package passed a CUDA A10G smoke check through the real TypeSafe Python SDK. Mixed Choice/Noul/Score requests used one backbone forward and generated zero tokens; plain JSON and zstd produced identical answers. Routing before and after typed calls passed the strict same-job comparison. Historical probability differences are reported separately. All 303 saved routing decisions also replayed unchanged locally. See [API verification](evaluation/api-verification.json). This is interface and packaging evidence, not a new accuracy benchmark.
|
| 150 |
+
|
| 151 |
+
## Compute and serving
|
| 152 |
+
|
| 153 |
+
Reference execution remains the default. `--execution compiled` enables an optional layer-compiled CUDA path on the same weights. The completed 303-case regression preserved all final routing decisions, with a 31.1% warm median speed improvement on three measured workloads and a 0.9% allocated-memory increase. Internal shortlists and probability values can differ. Compilation time, throughput, production cost and general-workload parity are not established by that result. See [BENCHMARKS.md](BENCHMARKS.md).
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
The sequential 303-case run on one A10G had a median model time of 613 ms and p95 of 2,442 ms, excluding download, model loading, network and queueing. The first case took 50.6 seconds including cold kernel initialization. Peak allocated GPU memory was 10.01 GB and reserved memory was 10.45 GB (decimal units).
|
| 157 |
+
|
| 158 |
+
The reference server serializes requests. Concurrent throughput, production cost and smaller hardware have not been qualified. After downloading the package and installing dependencies, inference uses local files and requires no Hub access or external inference provider. Use a reverse proxy for TLS, rate limits and production observability.
|
| 159 |
+
|
| 160 |
+
## Limitations
|
| 161 |
+
|
| 162 |
+
- Ambiguous or contradictory definitions can produce disagreement or excess clarification.
|
| 163 |
+
- Missing conversation history is not reconstructed. Runtime examples are not actual dialogue turns.
|
| 164 |
+
- The routing resolver supports a documented subset of explicit priority syntax, not arbitrary workflow DSL execution.
|
| 165 |
+
- Routing cutoffs of 0.8 / 0.2 are policy settings, not correctness guarantees. Probability calibration must be checked on the intended workload.
|
| 166 |
+
- Ordered-score quality and long-context accuracy have not been independently benchmarked.
|
| 167 |
+
|
| 168 |
+
## License and reproducibility
|
| 169 |
+
|
| 170 |
+
Apache-2.0. Preserve [LICENSE](https://huggingface.co/murabcd/fev-4b/blob/main/LICENSE), [NOTICE](https://huggingface.co/murabcd/fev-4b/blob/main/NOTICE) and bundled upstream notices. [manifest.json](https://huggingface.co/murabcd/fev-4b/blob/main/manifest.json) records package hashes. Pin the `v0.2.0` tag or its commit when downloading. Weights are hosted on Hugging Face; source and usage examples are maintained on [GitHub](https://github.com/murabcd/fev).
|
README.md
CHANGED
|
@@ -20,8 +20,6 @@ Fev-4B is a decision model for classification, routing, triage and checks agains
|
|
| 20 |
|
| 21 |
Built on the Qwen3.5-4B architecture, Fev combines a native decision readout with a shared LoRA adapter. Categories, business rules, examples and conversation history are supplied at runtime: customer-specific retraining is not required to change them. The package also includes a two-stage conversation router that can return a clarification signal when no route is sufficiently supported. Reference execution uses one 24 GB NVIDIA GPU.
|
| 22 |
|
| 23 |
-
This card describes Fev-4B v0.2.0.
|
| 24 |
-
|
| 25 |
## Model details
|
| 26 |
|
| 27 |
| | |
|
|
@@ -37,7 +35,7 @@ This card describes Fev-4B v0.2.0.
|
|
| 37 |
| Languages | Russian |
|
| 38 |
| Reference hardware | NVIDIA A10G, 24 GB; CUDA BF16 |
|
| 39 |
| License | Apache-2.0; bundled notices retained |
|
| 40 |
-
| Version | Fev v0.2.
|
| 41 |
|
| 42 |
**Input.** Text, a JSON object or an array as the state, plus 1–64 named questions:
|
| 43 |
|
|
@@ -72,7 +70,7 @@ Download the complete versioned model package. On a Linux server with a compatib
|
|
| 72 |
|
| 73 |
```bash
|
| 74 |
uvx --from huggingface-hub==1.31.0 hf download murabcd/fev-4b \
|
| 75 |
-
--revision v0.2.
|
| 76 |
cd fev-4b
|
| 77 |
uv venv --python 3.12
|
| 78 |
uv pip install -r requirements.txt -r requirements-serve.txt
|
|
@@ -108,12 +106,22 @@ For the conversation-routing policy, use `Fev4B.route(...)` or `run.py --input e
|
|
| 108 |
|
| 109 |
## Training and release procedure
|
| 110 |
|
| 111 |
-
This release distributes a complete decision checkpoint, one shared adapter and inference code. The shared adapter comes from customer-agnostic synthetic decision research. No customer-specific adapter is selected at runtime
|
| 112 |
|
| 113 |
Checkpoint and adapter files retain their original precision and hashes. Exact ancestry, adapter hashes and changed-file boundaries are recorded in [provenance.json](provenance.json) and [CHANGES.md](CHANGES.md). This is an inference release, not a complete reproduction of historical training data or an independent audit of all upstream sources.
|
| 114 |
|
| 115 |
## Evaluation
|
| 116 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 117 |
The conversation-routing composition was measured on **303 development cases** with the full supplied business rules. Cases include synthetic and source-derived examples already consumed during development; this is not a fresh, independently human-labelled test set.
|
| 118 |
|
| 119 |
| Outcome | Cases |
|
|
@@ -142,6 +150,9 @@ These results measure the routing composition. They do not establish general typ
|
|
| 142 |
|
| 143 |
## Compute and serving
|
| 144 |
|
|
|
|
|
|
|
|
|
|
| 145 |
The sequential 303-case run on one A10G had a median model time of 613 ms and p95 of 2,442 ms, excluding download, model loading, network and queueing. The first case took 50.6 seconds including cold kernel initialization. Peak allocated GPU memory was 10.01 GB and reserved memory was 10.45 GB (decimal units).
|
| 146 |
|
| 147 |
The reference server serializes requests. Concurrent throughput, production cost and smaller hardware have not been qualified. After downloading the package and installing dependencies, inference uses local files and requires no Hub access or external inference provider. Use a reverse proxy for TLS, rate limits and production observability.
|
|
|
|
| 20 |
|
| 21 |
Built on the Qwen3.5-4B architecture, Fev combines a native decision readout with a shared LoRA adapter. Categories, business rules, examples and conversation history are supplied at runtime: customer-specific retraining is not required to change them. The package also includes a two-stage conversation router that can return a clarification signal when no route is sufficiently supported. Reference execution uses one 24 GB NVIDIA GPU.
|
| 22 |
|
|
|
|
|
|
|
| 23 |
## Model details
|
| 24 |
|
| 25 |
| | |
|
|
|
|
| 35 |
| Languages | Russian |
|
| 36 |
| Reference hardware | NVIDIA A10G, 24 GB; CUDA BF16 |
|
| 37 |
| License | Apache-2.0; bundled notices retained |
|
| 38 |
+
| Version | Fev v0.2.1 |
|
| 39 |
|
| 40 |
**Input.** Text, a JSON object or an array as the state, plus 1–64 named questions:
|
| 41 |
|
|
|
|
| 70 |
|
| 71 |
```bash
|
| 72 |
uvx --from huggingface-hub==1.31.0 hf download murabcd/fev-4b \
|
| 73 |
+
--revision v0.2.1 --local-dir fev-4b
|
| 74 |
cd fev-4b
|
| 75 |
uv venv --python 3.12
|
| 76 |
uv pip install -r requirements.txt -r requirements-serve.txt
|
|
|
|
| 106 |
|
| 107 |
## Training and release procedure
|
| 108 |
|
| 109 |
+
This release distributes a complete decision checkpoint, one shared adapter and inference code. The shared adapter comes from customer-agnostic synthetic decision research. No customer-specific adapter is selected at runtime. The original checkpoint and shared adapter remain the baseline; new training tools produce explicit research overlays rather than overwriting them. See [the training guide](TRAINING.md) and the installable `fev-finetune` skill for native decision cross-entropy, grouped data splits and separate temperature calibration. A toy smoke run verifies mechanics; it is not a new accuracy claim or an adopted checkpoint.
|
| 110 |
|
| 111 |
Checkpoint and adapter files retain their original precision and hashes. Exact ancestry, adapter hashes and changed-file boundaries are recorded in [provenance.json](provenance.json) and [CHANGES.md](CHANGES.md). This is an inference release, not a complete reproduction of historical training data or an independent audit of all upstream sources.
|
| 112 |
|
| 113 |
## Evaluation
|
| 114 |
|
| 115 |
+
The new common native typed-decision comparison against pinned Kev-4B and Jev is documented in [BENCHMARKS.md](BENCHMARKS.md), with separate public-development and authored-Russian denominators, probability metrics and reproducibility commands. It is distinct from the routing-composition evidence below.
|
| 116 |
+
|
| 117 |
+
| Native comparison | Fev-4B | Kev-4B | Jev-1.14.0 |
|
| 118 |
+
|---|---:|---:|---:|
|
| 119 |
+
| Public development, 764 questions | 83.6% | 80.2% | 84.6% |
|
| 120 |
+
| Authored Russian, 144 questions | 100.0% | 99.3% | 100.0% |
|
| 121 |
+
|
| 122 |
+
All three systems received the same inputs, with zero failed responses. Open-model measurements use their BF16 serving paths; Jev uses its pinned hosted API. The public suite is already published development data, and Russian probes repeat templates. These are engineering comparisons, not unseen-customer accuracy. No comparison inputs were used by the new trainer.
|
| 123 |
+
|
| 124 |
+
|
| 125 |
The conversation-routing composition was measured on **303 development cases** with the full supplied business rules. Cases include synthetic and source-derived examples already consumed during development; this is not a fresh, independently human-labelled test set.
|
| 126 |
|
| 127 |
| Outcome | Cases |
|
|
|
|
| 150 |
|
| 151 |
## Compute and serving
|
| 152 |
|
| 153 |
+
Reference execution remains the default. `--execution compiled` enables an optional layer-compiled CUDA path on the same weights. The completed 303-case regression preserved all final routing decisions, with a 31.1% warm median speed improvement on three measured workloads and a 0.9% allocated-memory increase. Internal shortlists and probability values can differ. Compilation time, throughput, production cost and general-workload parity are not established by that result. See [BENCHMARKS.md](BENCHMARKS.md).
|
| 154 |
+
|
| 155 |
+
|
| 156 |
The sequential 303-case run on one A10G had a median model time of 613 ms and p95 of 2,442 ms, excluding download, model loading, network and queueing. The first case took 50.6 seconds including cold kernel initialization. Peak allocated GPU memory was 10.01 GB and reserved memory was 10.45 GB (decimal units).
|
| 157 |
|
| 158 |
The reference server serializes requests. Concurrent throughput, production cost and smaller hardware have not been qualified. After downloading the package and installing dependencies, inference uses local files and requires no Hub access or external inference provider. Use a reverse proxy for TLS, rate limits and production observability.
|
RUNNING.md
CHANGED
|
@@ -68,4 +68,16 @@ Do not run with Python `-O`; native research invariants include assertions. This
|
|
| 68 |
|
| 69 |
## Typed decisions and HTTP
|
| 70 |
|
| 71 |
-
`Fev4B.decide` exposes Choice, Noul and Score without the routing policy. Use `serve.py` for the text-only System One endpoint; see [API.md](API.md). The
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
|
| 69 |
## Typed decisions and HTTP
|
| 70 |
|
| 71 |
+
`Fev4B.decide` exposes Choice, Noul and Score without the routing policy. Use `serve.py` for the text-only System One endpoint; see [API.md](API.md). The typed interface scores the supplied questions without applying the conversation-routing policy.
|
| 72 |
+
|
| 73 |
+
## Execution mode and explicit adapters
|
| 74 |
+
|
| 75 |
+
`--execution reference` is the default. `--execution compiled` enables only the qualified layer-compilation options; kernels are compiled lazily. It retains the BF16 backbone and FP32 unmerged adapter. Warm speed measurements do not include compilation and are not throughput guarantees. See [BENCHMARKS.md](BENCHMARKS.md) for the exact tested workloads and probability/parity limits.
|
| 76 |
+
|
| 77 |
+
The Python constructor accepts the same selection:
|
| 78 |
+
|
| 79 |
+
```python
|
| 80 |
+
model = Fev4B("models/fev-4b", execution="compiled")
|
| 81 |
+
```
|
| 82 |
+
|
| 83 |
+
A research adapter is loaded only with explicit `adaptation_dir=...` or `--adaptation-dir ...`. It is checked against the base identity and payload hashes before use. Reference weights are never overwritten. Newly trained or calibrated overlays need their own routing-quality and compiled-parity checks; the released checkpoint's results do not transfer automatically. See [TRAINING.md](TRAINING.md).
|
TRAINING.md
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Fine-tuning Fev
|
| 2 |
+
|
| 3 |
+
The complete [training guide](skills/fev-finetune/references/training.md) covers labelled typed decisions, grouped splits, the shared adapter trainer, separate temperature calibration, evaluation and bounded Hugging Face GPU jobs.
|
| 4 |
+
|
| 5 |
+
Install the companion agent skill after this revision is published:
|
| 6 |
+
|
| 7 |
+
```bash
|
| 8 |
+
npx skills add murabcd/fev@fev-finetune
|
| 9 |
+
```
|
| 10 |
+
|
| 11 |
+
It teaches agents to use Fev's native decision training workflow, preserve the base package, separate evaluation data and avoid per-customer retraining. Installing a skill does not train or publish a model.
|
evaluation/benchmarks/fev/predictions.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
evaluation/benchmarks/fev/report.json
ADDED
|
@@ -0,0 +1,675 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"overall": {
|
| 3 |
+
"questions": 908,
|
| 4 |
+
"answered": 908,
|
| 5 |
+
"errors": 0,
|
| 6 |
+
"correct": 783,
|
| 7 |
+
"accuracyAllQuestions": 0.8623348017621145,
|
| 8 |
+
"brierAnswered": 0.19616500539754675,
|
| 9 |
+
"nllAnswered": 0.374025080065555,
|
| 10 |
+
"ece10Answered": 0.04207503699664424,
|
| 11 |
+
"confidentWrongAt09": 26,
|
| 12 |
+
"coverageByFixedThreshold": [
|
| 13 |
+
{
|
| 14 |
+
"threshold": 0.5,
|
| 15 |
+
"accepted": 889,
|
| 16 |
+
"coverage": 0.9790748898678414,
|
| 17 |
+
"errorRate": 0.12373453318335208
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"threshold": 0.7,
|
| 21 |
+
"accepted": 819,
|
| 22 |
+
"coverage": 0.9019823788546255,
|
| 23 |
+
"errorRate": 0.08424908424908426
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"threshold": 0.8,
|
| 27 |
+
"accepted": 767,
|
| 28 |
+
"coverage": 0.8447136563876652,
|
| 29 |
+
"errorRate": 0.06649282920469361
|
| 30 |
+
},
|
| 31 |
+
{
|
| 32 |
+
"threshold": 0.9,
|
| 33 |
+
"accepted": 676,
|
| 34 |
+
"coverage": 0.7444933920704846,
|
| 35 |
+
"errorRate": 0.038461538461538464
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"threshold": 0.95,
|
| 39 |
+
"accepted": 567,
|
| 40 |
+
"coverage": 0.6244493392070485,
|
| 41 |
+
"errorRate": 0.012345679012345678
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"threshold": 0.99,
|
| 45 |
+
"accepted": 0,
|
| 46 |
+
"coverage": 0.0,
|
| 47 |
+
"errorRate": null
|
| 48 |
+
}
|
| 49 |
+
],
|
| 50 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 51 |
+
},
|
| 52 |
+
"byType": {
|
| 53 |
+
"choice": {
|
| 54 |
+
"questions": 492,
|
| 55 |
+
"answered": 492,
|
| 56 |
+
"errors": 0,
|
| 57 |
+
"correct": 400,
|
| 58 |
+
"accuracyAllQuestions": 0.8130081300813008,
|
| 59 |
+
"brierAnswered": 0.2615263089500927,
|
| 60 |
+
"nllAnswered": 0.5247360934156637,
|
| 61 |
+
"ece10Answered": 0.06533461933177653,
|
| 62 |
+
"confidentWrongAt09": 20,
|
| 63 |
+
"coverageByFixedThreshold": [
|
| 64 |
+
{
|
| 65 |
+
"threshold": 0.5,
|
| 66 |
+
"accepted": 473,
|
| 67 |
+
"coverage": 0.9613821138211383,
|
| 68 |
+
"errorRate": 0.16279069767441862
|
| 69 |
+
},
|
| 70 |
+
{
|
| 71 |
+
"threshold": 0.7,
|
| 72 |
+
"accepted": 424,
|
| 73 |
+
"coverage": 0.8617886178861789,
|
| 74 |
+
"errorRate": 0.10377358490566038
|
| 75 |
+
},
|
| 76 |
+
{
|
| 77 |
+
"threshold": 0.8,
|
| 78 |
+
"accepted": 386,
|
| 79 |
+
"coverage": 0.7845528455284553,
|
| 80 |
+
"errorRate": 0.08031088082901554
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"threshold": 0.9,
|
| 84 |
+
"accepted": 331,
|
| 85 |
+
"coverage": 0.6727642276422764,
|
| 86 |
+
"errorRate": 0.06042296072507553
|
| 87 |
+
},
|
| 88 |
+
{
|
| 89 |
+
"threshold": 0.95,
|
| 90 |
+
"accepted": 267,
|
| 91 |
+
"coverage": 0.5426829268292683,
|
| 92 |
+
"errorRate": 0.026217228464419477
|
| 93 |
+
},
|
| 94 |
+
{
|
| 95 |
+
"threshold": 0.99,
|
| 96 |
+
"accepted": 0,
|
| 97 |
+
"coverage": 0.0,
|
| 98 |
+
"errorRate": null
|
| 99 |
+
}
|
| 100 |
+
],
|
| 101 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 102 |
+
},
|
| 103 |
+
"noul": {
|
| 104 |
+
"questions": 328,
|
| 105 |
+
"answered": 328,
|
| 106 |
+
"errors": 0,
|
| 107 |
+
"correct": 298,
|
| 108 |
+
"accuracyAllQuestions": 0.9085365853658537,
|
| 109 |
+
"brierAnswered": 0.13871514597249474,
|
| 110 |
+
"nllAnswered": 0.22139737153892355,
|
| 111 |
+
"ece10Answered": 0.04994271263817423,
|
| 112 |
+
"confidentWrongAt09": 5,
|
| 113 |
+
"coverageByFixedThreshold": [
|
| 114 |
+
{
|
| 115 |
+
"threshold": 0.5,
|
| 116 |
+
"accepted": 328,
|
| 117 |
+
"coverage": 1.0,
|
| 118 |
+
"errorRate": 0.09146341463414634
|
| 119 |
+
},
|
| 120 |
+
{
|
| 121 |
+
"threshold": 0.7,
|
| 122 |
+
"accepted": 309,
|
| 123 |
+
"coverage": 0.9420731707317073,
|
| 124 |
+
"errorRate": 0.07766990291262135
|
| 125 |
+
},
|
| 126 |
+
{
|
| 127 |
+
"threshold": 0.8,
|
| 128 |
+
"accepted": 296,
|
| 129 |
+
"coverage": 0.9024390243902439,
|
| 130 |
+
"errorRate": 0.06418918918918919
|
| 131 |
+
},
|
| 132 |
+
{
|
| 133 |
+
"threshold": 0.9,
|
| 134 |
+
"accepted": 261,
|
| 135 |
+
"coverage": 0.7957317073170732,
|
| 136 |
+
"errorRate": 0.019157088122605363
|
| 137 |
+
},
|
| 138 |
+
{
|
| 139 |
+
"threshold": 0.95,
|
| 140 |
+
"accepted": 220,
|
| 141 |
+
"coverage": 0.6707317073170732,
|
| 142 |
+
"errorRate": 0.0
|
| 143 |
+
},
|
| 144 |
+
{
|
| 145 |
+
"threshold": 0.99,
|
| 146 |
+
"accepted": 0,
|
| 147 |
+
"coverage": 0.0,
|
| 148 |
+
"errorRate": null
|
| 149 |
+
}
|
| 150 |
+
],
|
| 151 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 152 |
+
},
|
| 153 |
+
"score": {
|
| 154 |
+
"questions": 88,
|
| 155 |
+
"answered": 88,
|
| 156 |
+
"errors": 0,
|
| 157 |
+
"correct": 85,
|
| 158 |
+
"accuracyAllQuestions": 0.9659090909090909,
|
| 159 |
+
"brierAnswered": 0.04486719339259727,
|
| 160 |
+
"nllAnswered": 0.10029860084375529,
|
| 161 |
+
"ece10Answered": 0.03799778408573792,
|
| 162 |
+
"confidentWrongAt09": 1,
|
| 163 |
+
"coverageByFixedThreshold": [
|
| 164 |
+
{
|
| 165 |
+
"threshold": 0.5,
|
| 166 |
+
"accepted": 88,
|
| 167 |
+
"coverage": 1.0,
|
| 168 |
+
"errorRate": 0.03409090909090909
|
| 169 |
+
},
|
| 170 |
+
{
|
| 171 |
+
"threshold": 0.7,
|
| 172 |
+
"accepted": 86,
|
| 173 |
+
"coverage": 0.9772727272727273,
|
| 174 |
+
"errorRate": 0.011627906976744186
|
| 175 |
+
},
|
| 176 |
+
{
|
| 177 |
+
"threshold": 0.8,
|
| 178 |
+
"accepted": 85,
|
| 179 |
+
"coverage": 0.9659090909090909,
|
| 180 |
+
"errorRate": 0.011764705882352941
|
| 181 |
+
},
|
| 182 |
+
{
|
| 183 |
+
"threshold": 0.9,
|
| 184 |
+
"accepted": 84,
|
| 185 |
+
"coverage": 0.9545454545454546,
|
| 186 |
+
"errorRate": 0.011904761904761904
|
| 187 |
+
},
|
| 188 |
+
{
|
| 189 |
+
"threshold": 0.95,
|
| 190 |
+
"accepted": 80,
|
| 191 |
+
"coverage": 0.9090909090909091,
|
| 192 |
+
"errorRate": 0.0
|
| 193 |
+
},
|
| 194 |
+
{
|
| 195 |
+
"threshold": 0.99,
|
| 196 |
+
"accepted": 0,
|
| 197 |
+
"coverage": 0.0,
|
| 198 |
+
"errorRate": null
|
| 199 |
+
}
|
| 200 |
+
],
|
| 201 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 202 |
+
}
|
| 203 |
+
},
|
| 204 |
+
"bySource": {
|
| 205 |
+
"authored-russian-typed": {
|
| 206 |
+
"questions": 144,
|
| 207 |
+
"answered": 144,
|
| 208 |
+
"errors": 0,
|
| 209 |
+
"correct": 144,
|
| 210 |
+
"accuracyAllQuestions": 1.0,
|
| 211 |
+
"brierAnswered": 0.0009998268084272204,
|
| 212 |
+
"nllAnswered": 0.024466212500916604,
|
| 213 |
+
"ece10Answered": 0.024139216454452272,
|
| 214 |
+
"confidentWrongAt09": 0,
|
| 215 |
+
"coverageByFixedThreshold": [
|
| 216 |
+
{
|
| 217 |
+
"threshold": 0.5,
|
| 218 |
+
"accepted": 144,
|
| 219 |
+
"coverage": 1.0,
|
| 220 |
+
"errorRate": 0.0
|
| 221 |
+
},
|
| 222 |
+
{
|
| 223 |
+
"threshold": 0.7,
|
| 224 |
+
"accepted": 144,
|
| 225 |
+
"coverage": 1.0,
|
| 226 |
+
"errorRate": 0.0
|
| 227 |
+
},
|
| 228 |
+
{
|
| 229 |
+
"threshold": 0.8,
|
| 230 |
+
"accepted": 144,
|
| 231 |
+
"coverage": 1.0,
|
| 232 |
+
"errorRate": 0.0
|
| 233 |
+
},
|
| 234 |
+
{
|
| 235 |
+
"threshold": 0.9,
|
| 236 |
+
"accepted": 144,
|
| 237 |
+
"coverage": 1.0,
|
| 238 |
+
"errorRate": 0.0
|
| 239 |
+
},
|
| 240 |
+
{
|
| 241 |
+
"threshold": 0.95,
|
| 242 |
+
"accepted": 144,
|
| 243 |
+
"coverage": 1.0,
|
| 244 |
+
"errorRate": 0.0
|
| 245 |
+
},
|
| 246 |
+
{
|
| 247 |
+
"threshold": 0.99,
|
| 248 |
+
"accepted": 0,
|
| 249 |
+
"coverage": 0.0,
|
| 250 |
+
"errorRate": null
|
| 251 |
+
}
|
| 252 |
+
],
|
| 253 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 254 |
+
},
|
| 255 |
+
"composition_holdout": {
|
| 256 |
+
"questions": 96,
|
| 257 |
+
"answered": 96,
|
| 258 |
+
"errors": 0,
|
| 259 |
+
"correct": 84,
|
| 260 |
+
"accuracyAllQuestions": 0.875,
|
| 261 |
+
"brierAnswered": 0.18821988619559113,
|
| 262 |
+
"nllAnswered": 0.31506340814867745,
|
| 263 |
+
"ece10Answered": 0.08715274769631735,
|
| 264 |
+
"confidentWrongAt09": 9,
|
| 265 |
+
"coverageByFixedThreshold": [
|
| 266 |
+
{
|
| 267 |
+
"threshold": 0.5,
|
| 268 |
+
"accepted": 96,
|
| 269 |
+
"coverage": 1.0,
|
| 270 |
+
"errorRate": 0.125
|
| 271 |
+
},
|
| 272 |
+
{
|
| 273 |
+
"threshold": 0.7,
|
| 274 |
+
"accepted": 92,
|
| 275 |
+
"coverage": 0.9583333333333334,
|
| 276 |
+
"errorRate": 0.09782608695652174
|
| 277 |
+
},
|
| 278 |
+
{
|
| 279 |
+
"threshold": 0.8,
|
| 280 |
+
"accepted": 92,
|
| 281 |
+
"coverage": 0.9583333333333334,
|
| 282 |
+
"errorRate": 0.09782608695652174
|
| 283 |
+
},
|
| 284 |
+
{
|
| 285 |
+
"threshold": 0.9,
|
| 286 |
+
"accepted": 92,
|
| 287 |
+
"coverage": 0.9583333333333334,
|
| 288 |
+
"errorRate": 0.09782608695652174
|
| 289 |
+
},
|
| 290 |
+
{
|
| 291 |
+
"threshold": 0.95,
|
| 292 |
+
"accepted": 86,
|
| 293 |
+
"coverage": 0.8958333333333334,
|
| 294 |
+
"errorRate": 0.03488372093023256
|
| 295 |
+
},
|
| 296 |
+
{
|
| 297 |
+
"threshold": 0.99,
|
| 298 |
+
"accepted": 0,
|
| 299 |
+
"coverage": 0.0,
|
| 300 |
+
"errorRate": null
|
| 301 |
+
}
|
| 302 |
+
],
|
| 303 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 304 |
+
},
|
| 305 |
+
"emotion": {
|
| 306 |
+
"questions": 116,
|
| 307 |
+
"answered": 116,
|
| 308 |
+
"errors": 0,
|
| 309 |
+
"correct": 69,
|
| 310 |
+
"accuracyAllQuestions": 0.5948275862068966,
|
| 311 |
+
"brierAnswered": 0.5245559470753833,
|
| 312 |
+
"nllAnswered": 1.0208149336604613,
|
| 313 |
+
"ece10Answered": 0.12016921458494689,
|
| 314 |
+
"confidentWrongAt09": 2,
|
| 315 |
+
"coverageByFixedThreshold": [
|
| 316 |
+
{
|
| 317 |
+
"threshold": 0.5,
|
| 318 |
+
"accepted": 101,
|
| 319 |
+
"coverage": 0.8706896551724138,
|
| 320 |
+
"errorRate": 0.3564356435643564
|
| 321 |
+
},
|
| 322 |
+
{
|
| 323 |
+
"threshold": 0.7,
|
| 324 |
+
"accepted": 70,
|
| 325 |
+
"coverage": 0.603448275862069,
|
| 326 |
+
"errorRate": 0.22857142857142856
|
| 327 |
+
},
|
| 328 |
+
{
|
| 329 |
+
"threshold": 0.8,
|
| 330 |
+
"accepted": 44,
|
| 331 |
+
"coverage": 0.3793103448275862,
|
| 332 |
+
"errorRate": 0.18181818181818182
|
| 333 |
+
},
|
| 334 |
+
{
|
| 335 |
+
"threshold": 0.9,
|
| 336 |
+
"accepted": 8,
|
| 337 |
+
"coverage": 0.06896551724137931,
|
| 338 |
+
"errorRate": 0.25
|
| 339 |
+
},
|
| 340 |
+
{
|
| 341 |
+
"threshold": 0.95,
|
| 342 |
+
"accepted": 0,
|
| 343 |
+
"coverage": 0.0,
|
| 344 |
+
"errorRate": null
|
| 345 |
+
},
|
| 346 |
+
{
|
| 347 |
+
"threshold": 0.99,
|
| 348 |
+
"accepted": 0,
|
| 349 |
+
"coverage": 0.0,
|
| 350 |
+
"errorRate": null
|
| 351 |
+
}
|
| 352 |
+
],
|
| 353 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 354 |
+
},
|
| 355 |
+
"legacy_holdout": {
|
| 356 |
+
"questions": 80,
|
| 357 |
+
"answered": 80,
|
| 358 |
+
"errors": 0,
|
| 359 |
+
"correct": 77,
|
| 360 |
+
"accuracyAllQuestions": 0.9625,
|
| 361 |
+
"brierAnswered": 0.04910351545555701,
|
| 362 |
+
"nllAnswered": 0.10416551178163372,
|
| 363 |
+
"ece10Answered": 0.03574970703743059,
|
| 364 |
+
"confidentWrongAt09": 1,
|
| 365 |
+
"coverageByFixedThreshold": [
|
| 366 |
+
{
|
| 367 |
+
"threshold": 0.5,
|
| 368 |
+
"accepted": 80,
|
| 369 |
+
"coverage": 1.0,
|
| 370 |
+
"errorRate": 0.0375
|
| 371 |
+
},
|
| 372 |
+
{
|
| 373 |
+
"threshold": 0.7,
|
| 374 |
+
"accepted": 78,
|
| 375 |
+
"coverage": 0.975,
|
| 376 |
+
"errorRate": 0.01282051282051282
|
| 377 |
+
},
|
| 378 |
+
{
|
| 379 |
+
"threshold": 0.8,
|
| 380 |
+
"accepted": 77,
|
| 381 |
+
"coverage": 0.9625,
|
| 382 |
+
"errorRate": 0.012987012987012988
|
| 383 |
+
},
|
| 384 |
+
{
|
| 385 |
+
"threshold": 0.9,
|
| 386 |
+
"accepted": 76,
|
| 387 |
+
"coverage": 0.95,
|
| 388 |
+
"errorRate": 0.013157894736842105
|
| 389 |
+
},
|
| 390 |
+
{
|
| 391 |
+
"threshold": 0.95,
|
| 392 |
+
"accepted": 72,
|
| 393 |
+
"coverage": 0.9,
|
| 394 |
+
"errorRate": 0.0
|
| 395 |
+
},
|
| 396 |
+
{
|
| 397 |
+
"threshold": 0.99,
|
| 398 |
+
"accepted": 0,
|
| 399 |
+
"coverage": 0.0,
|
| 400 |
+
"errorRate": null
|
| 401 |
+
}
|
| 402 |
+
],
|
| 403 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 404 |
+
},
|
| 405 |
+
"mmlu": {
|
| 406 |
+
"questions": 116,
|
| 407 |
+
"answered": 116,
|
| 408 |
+
"errors": 0,
|
| 409 |
+
"correct": 86,
|
| 410 |
+
"accuracyAllQuestions": 0.7413793103448276,
|
| 411 |
+
"brierAnswered": 0.39416899029285185,
|
| 412 |
+
"nllAnswered": 0.8394104030580065,
|
| 413 |
+
"ece10Answered": 0.12528513793721022,
|
| 414 |
+
"confidentWrongAt09": 9,
|
| 415 |
+
"coverageByFixedThreshold": [
|
| 416 |
+
{
|
| 417 |
+
"threshold": 0.5,
|
| 418 |
+
"accepted": 112,
|
| 419 |
+
"coverage": 0.9655172413793104,
|
| 420 |
+
"errorRate": 0.23214285714285715
|
| 421 |
+
},
|
| 422 |
+
{
|
| 423 |
+
"threshold": 0.7,
|
| 424 |
+
"accepted": 100,
|
| 425 |
+
"coverage": 0.8620689655172413,
|
| 426 |
+
"errorRate": 0.18
|
| 427 |
+
},
|
| 428 |
+
{
|
| 429 |
+
"threshold": 0.8,
|
| 430 |
+
"accepted": 91,
|
| 431 |
+
"coverage": 0.7844827586206896,
|
| 432 |
+
"errorRate": 0.14285714285714285
|
| 433 |
+
},
|
| 434 |
+
{
|
| 435 |
+
"threshold": 0.9,
|
| 436 |
+
"accepted": 77,
|
| 437 |
+
"coverage": 0.6637931034482759,
|
| 438 |
+
"errorRate": 0.11688311688311688
|
| 439 |
+
},
|
| 440 |
+
{
|
| 441 |
+
"threshold": 0.95,
|
| 442 |
+
"accepted": 42,
|
| 443 |
+
"coverage": 0.3620689655172414,
|
| 444 |
+
"errorRate": 0.09523809523809523
|
| 445 |
+
},
|
| 446 |
+
{
|
| 447 |
+
"threshold": 0.99,
|
| 448 |
+
"accepted": 0,
|
| 449 |
+
"coverage": 0.0,
|
| 450 |
+
"errorRate": null
|
| 451 |
+
}
|
| 452 |
+
],
|
| 453 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 454 |
+
},
|
| 455 |
+
"paws": {
|
| 456 |
+
"questions": 80,
|
| 457 |
+
"answered": 80,
|
| 458 |
+
"errors": 0,
|
| 459 |
+
"correct": 68,
|
| 460 |
+
"accuracyAllQuestions": 0.85,
|
| 461 |
+
"brierAnswered": 0.20695251414059662,
|
| 462 |
+
"nllAnswered": 0.3140944708626177,
|
| 463 |
+
"ece10Answered": 0.08749262200436604,
|
| 464 |
+
"confidentWrongAt09": 2,
|
| 465 |
+
"coverageByFixedThreshold": [
|
| 466 |
+
{
|
| 467 |
+
"threshold": 0.5,
|
| 468 |
+
"accepted": 80,
|
| 469 |
+
"coverage": 1.0,
|
| 470 |
+
"errorRate": 0.15
|
| 471 |
+
},
|
| 472 |
+
{
|
| 473 |
+
"threshold": 0.7,
|
| 474 |
+
"accepted": 75,
|
| 475 |
+
"coverage": 0.9375,
|
| 476 |
+
"errorRate": 0.12
|
| 477 |
+
},
|
| 478 |
+
{
|
| 479 |
+
"threshold": 0.8,
|
| 480 |
+
"accepted": 73,
|
| 481 |
+
"coverage": 0.9125,
|
| 482 |
+
"errorRate": 0.0958904109589041
|
| 483 |
+
},
|
| 484 |
+
{
|
| 485 |
+
"threshold": 0.9,
|
| 486 |
+
"accepted": 65,
|
| 487 |
+
"coverage": 0.8125,
|
| 488 |
+
"errorRate": 0.03076923076923077
|
| 489 |
+
},
|
| 490 |
+
{
|
| 491 |
+
"threshold": 0.95,
|
| 492 |
+
"accepted": 58,
|
| 493 |
+
"coverage": 0.725,
|
| 494 |
+
"errorRate": 0.0
|
| 495 |
+
},
|
| 496 |
+
{
|
| 497 |
+
"threshold": 0.99,
|
| 498 |
+
"accepted": 0,
|
| 499 |
+
"coverage": 0.0,
|
| 500 |
+
"errorRate": null
|
| 501 |
+
}
|
| 502 |
+
],
|
| 503 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 504 |
+
},
|
| 505 |
+
"qnli": {
|
| 506 |
+
"questions": 80,
|
| 507 |
+
"answered": 80,
|
| 508 |
+
"errors": 0,
|
| 509 |
+
"correct": 74,
|
| 510 |
+
"accuracyAllQuestions": 0.925,
|
| 511 |
+
"brierAnswered": 0.1185264487594759,
|
| 512 |
+
"nllAnswered": 0.19157792439378024,
|
| 513 |
+
"ece10Answered": 0.07130364771118254,
|
| 514 |
+
"confidentWrongAt09": 1,
|
| 515 |
+
"coverageByFixedThreshold": [
|
| 516 |
+
{
|
| 517 |
+
"threshold": 0.5,
|
| 518 |
+
"accepted": 80,
|
| 519 |
+
"coverage": 1.0,
|
| 520 |
+
"errorRate": 0.075
|
| 521 |
+
},
|
| 522 |
+
{
|
| 523 |
+
"threshold": 0.7,
|
| 524 |
+
"accepted": 79,
|
| 525 |
+
"coverage": 0.9875,
|
| 526 |
+
"errorRate": 0.0759493670886076
|
| 527 |
+
},
|
| 528 |
+
{
|
| 529 |
+
"threshold": 0.8,
|
| 530 |
+
"accepted": 77,
|
| 531 |
+
"coverage": 0.9625,
|
| 532 |
+
"errorRate": 0.06493506493506493
|
| 533 |
+
},
|
| 534 |
+
{
|
| 535 |
+
"threshold": 0.9,
|
| 536 |
+
"accepted": 71,
|
| 537 |
+
"coverage": 0.8875,
|
| 538 |
+
"errorRate": 0.014084507042253521
|
| 539 |
+
},
|
| 540 |
+
{
|
| 541 |
+
"threshold": 0.95,
|
| 542 |
+
"accepted": 60,
|
| 543 |
+
"coverage": 0.75,
|
| 544 |
+
"errorRate": 0.0
|
| 545 |
+
},
|
| 546 |
+
{
|
| 547 |
+
"threshold": 0.99,
|
| 548 |
+
"accepted": 0,
|
| 549 |
+
"coverage": 0.0,
|
| 550 |
+
"errorRate": null
|
| 551 |
+
}
|
| 552 |
+
],
|
| 553 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 554 |
+
},
|
| 555 |
+
"sciq": {
|
| 556 |
+
"questions": 116,
|
| 557 |
+
"answered": 116,
|
| 558 |
+
"errors": 0,
|
| 559 |
+
"correct": 113,
|
| 560 |
+
"accuracyAllQuestions": 0.9741379310344828,
|
| 561 |
+
"brierAnswered": 0.03409531064541125,
|
| 562 |
+
"nllAnswered": 0.09114362656929587,
|
| 563 |
+
"ece10Answered": 0.05454464085684041,
|
| 564 |
+
"confidentWrongAt09": 0,
|
| 565 |
+
"coverageByFixedThreshold": [
|
| 566 |
+
{
|
| 567 |
+
"threshold": 0.5,
|
| 568 |
+
"accepted": 116,
|
| 569 |
+
"coverage": 1.0,
|
| 570 |
+
"errorRate": 0.02586206896551724
|
| 571 |
+
},
|
| 572 |
+
{
|
| 573 |
+
"threshold": 0.7,
|
| 574 |
+
"accepted": 114,
|
| 575 |
+
"coverage": 0.9827586206896551,
|
| 576 |
+
"errorRate": 0.008771929824561403
|
| 577 |
+
},
|
| 578 |
+
{
|
| 579 |
+
"threshold": 0.8,
|
| 580 |
+
"accepted": 111,
|
| 581 |
+
"coverage": 0.9568965517241379,
|
| 582 |
+
"errorRate": 0.009009009009009009
|
| 583 |
+
},
|
| 584 |
+
{
|
| 585 |
+
"threshold": 0.9,
|
| 586 |
+
"accepted": 106,
|
| 587 |
+
"coverage": 0.9137931034482759,
|
| 588 |
+
"errorRate": 0.0
|
| 589 |
+
},
|
| 590 |
+
{
|
| 591 |
+
"threshold": 0.95,
|
| 592 |
+
"accepted": 91,
|
| 593 |
+
"coverage": 0.7844827586206896,
|
| 594 |
+
"errorRate": 0.0
|
| 595 |
+
},
|
| 596 |
+
{
|
| 597 |
+
"threshold": 0.99,
|
| 598 |
+
"accepted": 0,
|
| 599 |
+
"coverage": 0.0,
|
| 600 |
+
"errorRate": null
|
| 601 |
+
}
|
| 602 |
+
],
|
| 603 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 604 |
+
},
|
| 605 |
+
"tweet_offensive": {
|
| 606 |
+
"questions": 80,
|
| 607 |
+
"answered": 80,
|
| 608 |
+
"errors": 0,
|
| 609 |
+
"correct": 68,
|
| 610 |
+
"accuracyAllQuestions": 0.85,
|
| 611 |
+
"brierAnswered": 0.24263742159686036,
|
| 612 |
+
"nllAnswered": 0.38374648265869776,
|
| 613 |
+
"ece10Answered": 0.05576197181938556,
|
| 614 |
+
"confidentWrongAt09": 2,
|
| 615 |
+
"coverageByFixedThreshold": [
|
| 616 |
+
{
|
| 617 |
+
"threshold": 0.5,
|
| 618 |
+
"accepted": 80,
|
| 619 |
+
"coverage": 1.0,
|
| 620 |
+
"errorRate": 0.15
|
| 621 |
+
},
|
| 622 |
+
{
|
| 623 |
+
"threshold": 0.7,
|
| 624 |
+
"accepted": 67,
|
| 625 |
+
"coverage": 0.8375,
|
| 626 |
+
"errorRate": 0.13432835820895522
|
| 627 |
+
},
|
| 628 |
+
{
|
| 629 |
+
"threshold": 0.8,
|
| 630 |
+
"accepted": 58,
|
| 631 |
+
"coverage": 0.725,
|
| 632 |
+
"errorRate": 0.1206896551724138
|
| 633 |
+
},
|
| 634 |
+
{
|
| 635 |
+
"threshold": 0.9,
|
| 636 |
+
"accepted": 37,
|
| 637 |
+
"coverage": 0.4625,
|
| 638 |
+
"errorRate": 0.05405405405405406
|
| 639 |
+
},
|
| 640 |
+
{
|
| 641 |
+
"threshold": 0.95,
|
| 642 |
+
"accepted": 14,
|
| 643 |
+
"coverage": 0.175,
|
| 644 |
+
"errorRate": 0.0
|
| 645 |
+
},
|
| 646 |
+
{
|
| 647 |
+
"threshold": 0.99,
|
| 648 |
+
"accepted": 0,
|
| 649 |
+
"coverage": 0.0,
|
| 650 |
+
"errorRate": null
|
| 651 |
+
}
|
| 652 |
+
],
|
| 653 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 654 |
+
}
|
| 655 |
+
},
|
| 656 |
+
"provenance": {
|
| 657 |
+
"revision": "3ef077cd7cc3c97d696c18150358ab4de3a6884e",
|
| 658 |
+
"codeZipSha256": "b3b7808d7c2cbe2ce26d4b31ee0dde2b3e894c6776537931b61cd173ef526f8e",
|
| 659 |
+
"execution": "reference BF16 backbone, FP32 shared adapter unmerged",
|
| 660 |
+
"dataSha256": "9266d6ea1ce83c9755a60cf48c7469b0fa3264c229e96f4142d683bada8fef91",
|
| 661 |
+
"gpu": "NVIDIA A10G",
|
| 662 |
+
"torch": "2.14.0+cu130",
|
| 663 |
+
"questionIsolation": false
|
| 664 |
+
},
|
| 665 |
+
"servedModels": [
|
| 666 |
+
"fev-4b"
|
| 667 |
+
],
|
| 668 |
+
"records": 812,
|
| 669 |
+
"startedUtc": null,
|
| 670 |
+
"wallLatencyMs": {
|
| 671 |
+
"median": 86.00620499998968,
|
| 672 |
+
"p95": 102.34360900000183
|
| 673 |
+
},
|
| 674 |
+
"latencyMeaning": "Sequential wall time including client overhead/cold calls; not model-only time or throughput"
|
| 675 |
+
}
|
evaluation/benchmarks/fev/rows.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
evaluation/benchmarks/jev/predictions.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
evaluation/benchmarks/jev/report.json
ADDED
|
@@ -0,0 +1,678 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"overall": {
|
| 3 |
+
"questions": 908,
|
| 4 |
+
"answered": 908,
|
| 5 |
+
"errors": 0,
|
| 6 |
+
"correct": 790,
|
| 7 |
+
"accuracyAllQuestions": 0.8700440528634361,
|
| 8 |
+
"brierAnswered": 0.18933414586842034,
|
| 9 |
+
"nllAnswered": 0.8838910343827637,
|
| 10 |
+
"ece10Answered": 0.04661137809816233,
|
| 11 |
+
"confidentWrongAt09": 36,
|
| 12 |
+
"coverageByFixedThreshold": [
|
| 13 |
+
{
|
| 14 |
+
"threshold": 0.5,
|
| 15 |
+
"accepted": 900,
|
| 16 |
+
"coverage": 0.9911894273127754,
|
| 17 |
+
"errorRate": 0.12444444444444444
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"threshold": 0.7,
|
| 21 |
+
"accepted": 807,
|
| 22 |
+
"coverage": 0.8887665198237885,
|
| 23 |
+
"errorRate": 0.08178438661710037
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"threshold": 0.8,
|
| 27 |
+
"accepted": 757,
|
| 28 |
+
"coverage": 0.8337004405286343,
|
| 29 |
+
"errorRate": 0.06340819022457067
|
| 30 |
+
},
|
| 31 |
+
{
|
| 32 |
+
"threshold": 0.9,
|
| 33 |
+
"accepted": 695,
|
| 34 |
+
"coverage": 0.7654185022026432,
|
| 35 |
+
"errorRate": 0.051798561151079135
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"threshold": 0.95,
|
| 39 |
+
"accepted": 629,
|
| 40 |
+
"coverage": 0.6927312775330396,
|
| 41 |
+
"errorRate": 0.04292527821939587
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"threshold": 0.99,
|
| 45 |
+
"accepted": 443,
|
| 46 |
+
"coverage": 0.4878854625550661,
|
| 47 |
+
"errorRate": 0.040632054176072234
|
| 48 |
+
}
|
| 49 |
+
],
|
| 50 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 51 |
+
},
|
| 52 |
+
"byType": {
|
| 53 |
+
"choice": {
|
| 54 |
+
"questions": 492,
|
| 55 |
+
"answered": 492,
|
| 56 |
+
"errors": 0,
|
| 57 |
+
"correct": 414,
|
| 58 |
+
"accuracyAllQuestions": 0.8414634146341463,
|
| 59 |
+
"brierAnswered": 0.2359069196108245,
|
| 60 |
+
"nllAnswered": 1.4490444419682396,
|
| 61 |
+
"ece10Answered": 0.08126652705921,
|
| 62 |
+
"confidentWrongAt09": 28,
|
| 63 |
+
"coverageByFixedThreshold": [
|
| 64 |
+
{
|
| 65 |
+
"threshold": 0.5,
|
| 66 |
+
"accepted": 484,
|
| 67 |
+
"coverage": 0.983739837398374,
|
| 68 |
+
"errorRate": 0.1487603305785124
|
| 69 |
+
},
|
| 70 |
+
{
|
| 71 |
+
"threshold": 0.7,
|
| 72 |
+
"accepted": 437,
|
| 73 |
+
"coverage": 0.8882113821138211,
|
| 74 |
+
"errorRate": 0.10526315789473684
|
| 75 |
+
},
|
| 76 |
+
{
|
| 77 |
+
"threshold": 0.8,
|
| 78 |
+
"accepted": 411,
|
| 79 |
+
"coverage": 0.8353658536585366,
|
| 80 |
+
"errorRate": 0.0827250608272506
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"threshold": 0.9,
|
| 84 |
+
"accepted": 384,
|
| 85 |
+
"coverage": 0.7804878048780488,
|
| 86 |
+
"errorRate": 0.07291666666666667
|
| 87 |
+
},
|
| 88 |
+
{
|
| 89 |
+
"threshold": 0.95,
|
| 90 |
+
"accepted": 362,
|
| 91 |
+
"coverage": 0.7357723577235772,
|
| 92 |
+
"errorRate": 0.06906077348066299
|
| 93 |
+
},
|
| 94 |
+
{
|
| 95 |
+
"threshold": 0.99,
|
| 96 |
+
"accepted": 320,
|
| 97 |
+
"coverage": 0.6504065040650406,
|
| 98 |
+
"errorRate": 0.05625
|
| 99 |
+
}
|
| 100 |
+
],
|
| 101 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 102 |
+
},
|
| 103 |
+
"noul": {
|
| 104 |
+
"questions": 328,
|
| 105 |
+
"answered": 328,
|
| 106 |
+
"errors": 0,
|
| 107 |
+
"correct": 289,
|
| 108 |
+
"accuracyAllQuestions": 0.8810975609756098,
|
| 109 |
+
"brierAnswered": 0.16377439024390245,
|
| 110 |
+
"nllAnswered": 0.26158678411952796,
|
| 111 |
+
"ece10Answered": 0.015304878048780484,
|
| 112 |
+
"confidentWrongAt09": 8,
|
| 113 |
+
"coverageByFixedThreshold": [
|
| 114 |
+
{
|
| 115 |
+
"threshold": 0.5,
|
| 116 |
+
"accepted": 328,
|
| 117 |
+
"coverage": 1.0,
|
| 118 |
+
"errorRate": 0.11890243902439024
|
| 119 |
+
},
|
| 120 |
+
{
|
| 121 |
+
"threshold": 0.7,
|
| 122 |
+
"accepted": 285,
|
| 123 |
+
"coverage": 0.8689024390243902,
|
| 124 |
+
"errorRate": 0.06666666666666667
|
| 125 |
+
},
|
| 126 |
+
{
|
| 127 |
+
"threshold": 0.8,
|
| 128 |
+
"accepted": 262,
|
| 129 |
+
"coverage": 0.7987804878048781,
|
| 130 |
+
"errorRate": 0.05343511450381679
|
| 131 |
+
},
|
| 132 |
+
{
|
| 133 |
+
"threshold": 0.9,
|
| 134 |
+
"accepted": 229,
|
| 135 |
+
"coverage": 0.698170731707317,
|
| 136 |
+
"errorRate": 0.034934497816593885
|
| 137 |
+
},
|
| 138 |
+
{
|
| 139 |
+
"threshold": 0.95,
|
| 140 |
+
"accepted": 186,
|
| 141 |
+
"coverage": 0.5670731707317073,
|
| 142 |
+
"errorRate": 0.010752688172043012
|
| 143 |
+
},
|
| 144 |
+
{
|
| 145 |
+
"threshold": 0.99,
|
| 146 |
+
"accepted": 55,
|
| 147 |
+
"coverage": 0.1676829268292683,
|
| 148 |
+
"errorRate": 0.0
|
| 149 |
+
}
|
| 150 |
+
],
|
| 151 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 152 |
+
},
|
| 153 |
+
"score": {
|
| 154 |
+
"questions": 88,
|
| 155 |
+
"answered": 88,
|
| 156 |
+
"errors": 0,
|
| 157 |
+
"correct": 87,
|
| 158 |
+
"accuracyAllQuestions": 0.9886363636363636,
|
| 159 |
+
"brierAnswered": 0.02421818181818182,
|
| 160 |
+
"nllAnswered": 0.043667370226935225,
|
| 161 |
+
"ece10Answered": 0.03250000000000011,
|
| 162 |
+
"confidentWrongAt09": 0,
|
| 163 |
+
"coverageByFixedThreshold": [
|
| 164 |
+
{
|
| 165 |
+
"threshold": 0.5,
|
| 166 |
+
"accepted": 88,
|
| 167 |
+
"coverage": 1.0,
|
| 168 |
+
"errorRate": 0.011363636363636364
|
| 169 |
+
},
|
| 170 |
+
{
|
| 171 |
+
"threshold": 0.7,
|
| 172 |
+
"accepted": 85,
|
| 173 |
+
"coverage": 0.9659090909090909,
|
| 174 |
+
"errorRate": 0.011764705882352941
|
| 175 |
+
},
|
| 176 |
+
{
|
| 177 |
+
"threshold": 0.8,
|
| 178 |
+
"accepted": 84,
|
| 179 |
+
"coverage": 0.9545454545454546,
|
| 180 |
+
"errorRate": 0.0
|
| 181 |
+
},
|
| 182 |
+
{
|
| 183 |
+
"threshold": 0.9,
|
| 184 |
+
"accepted": 82,
|
| 185 |
+
"coverage": 0.9318181818181818,
|
| 186 |
+
"errorRate": 0.0
|
| 187 |
+
},
|
| 188 |
+
{
|
| 189 |
+
"threshold": 0.95,
|
| 190 |
+
"accepted": 81,
|
| 191 |
+
"coverage": 0.9204545454545454,
|
| 192 |
+
"errorRate": 0.0
|
| 193 |
+
},
|
| 194 |
+
{
|
| 195 |
+
"threshold": 0.99,
|
| 196 |
+
"accepted": 68,
|
| 197 |
+
"coverage": 0.7727272727272727,
|
| 198 |
+
"errorRate": 0.0
|
| 199 |
+
}
|
| 200 |
+
],
|
| 201 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 202 |
+
}
|
| 203 |
+
},
|
| 204 |
+
"bySource": {
|
| 205 |
+
"authored-russian-typed": {
|
| 206 |
+
"questions": 144,
|
| 207 |
+
"answered": 144,
|
| 208 |
+
"errors": 0,
|
| 209 |
+
"correct": 144,
|
| 210 |
+
"accuracyAllQuestions": 1.0,
|
| 211 |
+
"brierAnswered": 0.00011805555555555571,
|
| 212 |
+
"nllAnswered": 0.005099213409504898,
|
| 213 |
+
"ece10Answered": 0.005069444444444349,
|
| 214 |
+
"confidentWrongAt09": 0,
|
| 215 |
+
"coverageByFixedThreshold": [
|
| 216 |
+
{
|
| 217 |
+
"threshold": 0.5,
|
| 218 |
+
"accepted": 144,
|
| 219 |
+
"coverage": 1.0,
|
| 220 |
+
"errorRate": 0.0
|
| 221 |
+
},
|
| 222 |
+
{
|
| 223 |
+
"threshold": 0.7,
|
| 224 |
+
"accepted": 144,
|
| 225 |
+
"coverage": 1.0,
|
| 226 |
+
"errorRate": 0.0
|
| 227 |
+
},
|
| 228 |
+
{
|
| 229 |
+
"threshold": 0.8,
|
| 230 |
+
"accepted": 144,
|
| 231 |
+
"coverage": 1.0,
|
| 232 |
+
"errorRate": 0.0
|
| 233 |
+
},
|
| 234 |
+
{
|
| 235 |
+
"threshold": 0.9,
|
| 236 |
+
"accepted": 144,
|
| 237 |
+
"coverage": 1.0,
|
| 238 |
+
"errorRate": 0.0
|
| 239 |
+
},
|
| 240 |
+
{
|
| 241 |
+
"threshold": 0.95,
|
| 242 |
+
"accepted": 144,
|
| 243 |
+
"coverage": 1.0,
|
| 244 |
+
"errorRate": 0.0
|
| 245 |
+
},
|
| 246 |
+
{
|
| 247 |
+
"threshold": 0.99,
|
| 248 |
+
"accepted": 138,
|
| 249 |
+
"coverage": 0.9583333333333334,
|
| 250 |
+
"errorRate": 0.0
|
| 251 |
+
}
|
| 252 |
+
],
|
| 253 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 254 |
+
},
|
| 255 |
+
"composition_holdout": {
|
| 256 |
+
"questions": 96,
|
| 257 |
+
"answered": 96,
|
| 258 |
+
"errors": 0,
|
| 259 |
+
"correct": 86,
|
| 260 |
+
"accuracyAllQuestions": 0.8958333333333334,
|
| 261 |
+
"brierAnswered": 0.11480833333333335,
|
| 262 |
+
"nllAnswered": 0.16559070638097498,
|
| 263 |
+
"ece10Answered": 0.041458333333333354,
|
| 264 |
+
"confidentWrongAt09": 1,
|
| 265 |
+
"coverageByFixedThreshold": [
|
| 266 |
+
{
|
| 267 |
+
"threshold": 0.5,
|
| 268 |
+
"accepted": 96,
|
| 269 |
+
"coverage": 1.0,
|
| 270 |
+
"errorRate": 0.10416666666666667
|
| 271 |
+
},
|
| 272 |
+
{
|
| 273 |
+
"threshold": 0.7,
|
| 274 |
+
"accepted": 81,
|
| 275 |
+
"coverage": 0.84375,
|
| 276 |
+
"errorRate": 0.012345679012345678
|
| 277 |
+
},
|
| 278 |
+
{
|
| 279 |
+
"threshold": 0.8,
|
| 280 |
+
"accepted": 80,
|
| 281 |
+
"coverage": 0.8333333333333334,
|
| 282 |
+
"errorRate": 0.0125
|
| 283 |
+
},
|
| 284 |
+
{
|
| 285 |
+
"threshold": 0.9,
|
| 286 |
+
"accepted": 77,
|
| 287 |
+
"coverage": 0.8020833333333334,
|
| 288 |
+
"errorRate": 0.012987012987012988
|
| 289 |
+
},
|
| 290 |
+
{
|
| 291 |
+
"threshold": 0.95,
|
| 292 |
+
"accepted": 73,
|
| 293 |
+
"coverage": 0.7604166666666666,
|
| 294 |
+
"errorRate": 0.0
|
| 295 |
+
},
|
| 296 |
+
{
|
| 297 |
+
"threshold": 0.99,
|
| 298 |
+
"accepted": 65,
|
| 299 |
+
"coverage": 0.6770833333333334,
|
| 300 |
+
"errorRate": 0.0
|
| 301 |
+
}
|
| 302 |
+
],
|
| 303 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 304 |
+
},
|
| 305 |
+
"emotion": {
|
| 306 |
+
"questions": 116,
|
| 307 |
+
"answered": 116,
|
| 308 |
+
"errors": 0,
|
| 309 |
+
"correct": 70,
|
| 310 |
+
"accuracyAllQuestions": 0.603448275862069,
|
| 311 |
+
"brierAnswered": 0.5788285340341767,
|
| 312 |
+
"nllAnswered": 3.146503884340211,
|
| 313 |
+
"ece10Answered": 0.23599616858237546,
|
| 314 |
+
"confidentWrongAt09": 14,
|
| 315 |
+
"coverageByFixedThreshold": [
|
| 316 |
+
{
|
| 317 |
+
"threshold": 0.5,
|
| 318 |
+
"accepted": 110,
|
| 319 |
+
"coverage": 0.9482758620689655,
|
| 320 |
+
"errorRate": 0.37272727272727274
|
| 321 |
+
},
|
| 322 |
+
{
|
| 323 |
+
"threshold": 0.7,
|
| 324 |
+
"accepted": 91,
|
| 325 |
+
"coverage": 0.7844827586206896,
|
| 326 |
+
"errorRate": 0.31868131868131866
|
| 327 |
+
},
|
| 328 |
+
{
|
| 329 |
+
"threshold": 0.8,
|
| 330 |
+
"accepted": 75,
|
| 331 |
+
"coverage": 0.646551724137931,
|
| 332 |
+
"errorRate": 0.25333333333333335
|
| 333 |
+
},
|
| 334 |
+
{
|
| 335 |
+
"threshold": 0.9,
|
| 336 |
+
"accepted": 59,
|
| 337 |
+
"coverage": 0.5086206896551724,
|
| 338 |
+
"errorRate": 0.23728813559322035
|
| 339 |
+
},
|
| 340 |
+
{
|
| 341 |
+
"threshold": 0.95,
|
| 342 |
+
"accepted": 52,
|
| 343 |
+
"coverage": 0.4482758620689655,
|
| 344 |
+
"errorRate": 0.25
|
| 345 |
+
},
|
| 346 |
+
{
|
| 347 |
+
"threshold": 0.99,
|
| 348 |
+
"accepted": 39,
|
| 349 |
+
"coverage": 0.33620689655172414,
|
| 350 |
+
"errorRate": 0.1794871794871795
|
| 351 |
+
}
|
| 352 |
+
],
|
| 353 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 354 |
+
},
|
| 355 |
+
"legacy_holdout": {
|
| 356 |
+
"questions": 80,
|
| 357 |
+
"answered": 80,
|
| 358 |
+
"errors": 0,
|
| 359 |
+
"correct": 79,
|
| 360 |
+
"accuracyAllQuestions": 0.9875,
|
| 361 |
+
"brierAnswered": 0.02695,
|
| 362 |
+
"nllAnswered": 0.05636262223757969,
|
| 363 |
+
"ece10Answered": 0.043999999999999984,
|
| 364 |
+
"confidentWrongAt09": 0,
|
| 365 |
+
"coverageByFixedThreshold": [
|
| 366 |
+
{
|
| 367 |
+
"threshold": 0.5,
|
| 368 |
+
"accepted": 80,
|
| 369 |
+
"coverage": 1.0,
|
| 370 |
+
"errorRate": 0.0125
|
| 371 |
+
},
|
| 372 |
+
{
|
| 373 |
+
"threshold": 0.7,
|
| 374 |
+
"accepted": 77,
|
| 375 |
+
"coverage": 0.9625,
|
| 376 |
+
"errorRate": 0.012987012987012988
|
| 377 |
+
},
|
| 378 |
+
{
|
| 379 |
+
"threshold": 0.8,
|
| 380 |
+
"accepted": 76,
|
| 381 |
+
"coverage": 0.95,
|
| 382 |
+
"errorRate": 0.0
|
| 383 |
+
},
|
| 384 |
+
{
|
| 385 |
+
"threshold": 0.9,
|
| 386 |
+
"accepted": 74,
|
| 387 |
+
"coverage": 0.925,
|
| 388 |
+
"errorRate": 0.0
|
| 389 |
+
},
|
| 390 |
+
{
|
| 391 |
+
"threshold": 0.95,
|
| 392 |
+
"accepted": 73,
|
| 393 |
+
"coverage": 0.9125,
|
| 394 |
+
"errorRate": 0.0
|
| 395 |
+
},
|
| 396 |
+
{
|
| 397 |
+
"threshold": 0.99,
|
| 398 |
+
"accepted": 31,
|
| 399 |
+
"coverage": 0.3875,
|
| 400 |
+
"errorRate": 0.0
|
| 401 |
+
}
|
| 402 |
+
],
|
| 403 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 404 |
+
},
|
| 405 |
+
"mmlu": {
|
| 406 |
+
"questions": 116,
|
| 407 |
+
"answered": 116,
|
| 408 |
+
"errors": 0,
|
| 409 |
+
"correct": 97,
|
| 410 |
+
"accuracyAllQuestions": 0.8362068965517241,
|
| 411 |
+
"brierAnswered": 0.28328529741863073,
|
| 412 |
+
"nllAnswered": 2.795124001333949,
|
| 413 |
+
"ece10Answered": 0.12087251828631135,
|
| 414 |
+
"confidentWrongAt09": 13,
|
| 415 |
+
"coverageByFixedThreshold": [
|
| 416 |
+
{
|
| 417 |
+
"threshold": 0.5,
|
| 418 |
+
"accepted": 114,
|
| 419 |
+
"coverage": 0.9827586206896551,
|
| 420 |
+
"errorRate": 0.15789473684210525
|
| 421 |
+
},
|
| 422 |
+
{
|
| 423 |
+
"threshold": 0.7,
|
| 424 |
+
"accepted": 105,
|
| 425 |
+
"coverage": 0.9051724137931034,
|
| 426 |
+
"errorRate": 0.13333333333333333
|
| 427 |
+
},
|
| 428 |
+
{
|
| 429 |
+
"threshold": 0.8,
|
| 430 |
+
"accepted": 99,
|
| 431 |
+
"coverage": 0.853448275862069,
|
| 432 |
+
"errorRate": 0.13131313131313133
|
| 433 |
+
},
|
| 434 |
+
{
|
| 435 |
+
"threshold": 0.9,
|
| 436 |
+
"accepted": 93,
|
| 437 |
+
"coverage": 0.8017241379310345,
|
| 438 |
+
"errorRate": 0.13978494623655913
|
| 439 |
+
},
|
| 440 |
+
{
|
| 441 |
+
"threshold": 0.95,
|
| 442 |
+
"accepted": 82,
|
| 443 |
+
"coverage": 0.7068965517241379,
|
| 444 |
+
"errorRate": 0.14634146341463414
|
| 445 |
+
},
|
| 446 |
+
{
|
| 447 |
+
"threshold": 0.99,
|
| 448 |
+
"accepted": 67,
|
| 449 |
+
"coverage": 0.5775862068965517,
|
| 450 |
+
"errorRate": 0.16417910447761194
|
| 451 |
+
}
|
| 452 |
+
],
|
| 453 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 454 |
+
},
|
| 455 |
+
"paws": {
|
| 456 |
+
"questions": 80,
|
| 457 |
+
"answered": 80,
|
| 458 |
+
"errors": 0,
|
| 459 |
+
"correct": 63,
|
| 460 |
+
"accuracyAllQuestions": 0.7875,
|
| 461 |
+
"brierAnswered": 0.3104425,
|
| 462 |
+
"nllAnswered": 0.4745888407794543,
|
| 463 |
+
"ece10Answered": 0.0953749999999999,
|
| 464 |
+
"confidentWrongAt09": 6,
|
| 465 |
+
"coverageByFixedThreshold": [
|
| 466 |
+
{
|
| 467 |
+
"threshold": 0.5,
|
| 468 |
+
"accepted": 80,
|
| 469 |
+
"coverage": 1.0,
|
| 470 |
+
"errorRate": 0.2125
|
| 471 |
+
},
|
| 472 |
+
{
|
| 473 |
+
"threshold": 0.7,
|
| 474 |
+
"accepted": 70,
|
| 475 |
+
"coverage": 0.875,
|
| 476 |
+
"errorRate": 0.17142857142857143
|
| 477 |
+
},
|
| 478 |
+
{
|
| 479 |
+
"threshold": 0.8,
|
| 480 |
+
"accepted": 67,
|
| 481 |
+
"coverage": 0.8375,
|
| 482 |
+
"errorRate": 0.16417910447761194
|
| 483 |
+
},
|
| 484 |
+
{
|
| 485 |
+
"threshold": 0.9,
|
| 486 |
+
"accepted": 56,
|
| 487 |
+
"coverage": 0.7,
|
| 488 |
+
"errorRate": 0.10714285714285714
|
| 489 |
+
},
|
| 490 |
+
{
|
| 491 |
+
"threshold": 0.95,
|
| 492 |
+
"accepted": 39,
|
| 493 |
+
"coverage": 0.4875,
|
| 494 |
+
"errorRate": 0.05128205128205128
|
| 495 |
+
},
|
| 496 |
+
{
|
| 497 |
+
"threshold": 0.99,
|
| 498 |
+
"accepted": 0,
|
| 499 |
+
"coverage": 0.0,
|
| 500 |
+
"errorRate": null
|
| 501 |
+
}
|
| 502 |
+
],
|
| 503 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 504 |
+
},
|
| 505 |
+
"qnli": {
|
| 506 |
+
"questions": 80,
|
| 507 |
+
"answered": 80,
|
| 508 |
+
"errors": 0,
|
| 509 |
+
"correct": 73,
|
| 510 |
+
"accuracyAllQuestions": 0.9125,
|
| 511 |
+
"brierAnswered": 0.094585,
|
| 512 |
+
"nllAnswered": 0.1699941228795627,
|
| 513 |
+
"ece10Answered": 0.08575000000000002,
|
| 514 |
+
"confidentWrongAt09": 0,
|
| 515 |
+
"coverageByFixedThreshold": [
|
| 516 |
+
{
|
| 517 |
+
"threshold": 0.5,
|
| 518 |
+
"accepted": 80,
|
| 519 |
+
"coverage": 1.0,
|
| 520 |
+
"errorRate": 0.0875
|
| 521 |
+
},
|
| 522 |
+
{
|
| 523 |
+
"threshold": 0.7,
|
| 524 |
+
"accepted": 71,
|
| 525 |
+
"coverage": 0.8875,
|
| 526 |
+
"errorRate": 0.014084507042253521
|
| 527 |
+
},
|
| 528 |
+
{
|
| 529 |
+
"threshold": 0.8,
|
| 530 |
+
"accepted": 63,
|
| 531 |
+
"coverage": 0.7875,
|
| 532 |
+
"errorRate": 0.0
|
| 533 |
+
},
|
| 534 |
+
{
|
| 535 |
+
"threshold": 0.9,
|
| 536 |
+
"accepted": 55,
|
| 537 |
+
"coverage": 0.6875,
|
| 538 |
+
"errorRate": 0.0
|
| 539 |
+
},
|
| 540 |
+
{
|
| 541 |
+
"threshold": 0.95,
|
| 542 |
+
"accepted": 42,
|
| 543 |
+
"coverage": 0.525,
|
| 544 |
+
"errorRate": 0.0
|
| 545 |
+
},
|
| 546 |
+
{
|
| 547 |
+
"threshold": 0.99,
|
| 548 |
+
"accepted": 1,
|
| 549 |
+
"coverage": 0.0125,
|
| 550 |
+
"errorRate": 0.0
|
| 551 |
+
}
|
| 552 |
+
],
|
| 553 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 554 |
+
},
|
| 555 |
+
"sciq": {
|
| 556 |
+
"questions": 116,
|
| 557 |
+
"answered": 116,
|
| 558 |
+
"errors": 0,
|
| 559 |
+
"correct": 113,
|
| 560 |
+
"accuracyAllQuestions": 0.9741379310344828,
|
| 561 |
+
"brierAnswered": 0.04341551724137931,
|
| 562 |
+
"nllAnswered": 0.0658923924477742,
|
| 563 |
+
"ece10Answered": 0.01767241379310347,
|
| 564 |
+
"confidentWrongAt09": 0,
|
| 565 |
+
"coverageByFixedThreshold": [
|
| 566 |
+
{
|
| 567 |
+
"threshold": 0.5,
|
| 568 |
+
"accepted": 116,
|
| 569 |
+
"coverage": 1.0,
|
| 570 |
+
"errorRate": 0.02586206896551724
|
| 571 |
+
},
|
| 572 |
+
{
|
| 573 |
+
"threshold": 0.7,
|
| 574 |
+
"accepted": 112,
|
| 575 |
+
"coverage": 0.9655172413793104,
|
| 576 |
+
"errorRate": 0.017857142857142856
|
| 577 |
+
},
|
| 578 |
+
{
|
| 579 |
+
"threshold": 0.8,
|
| 580 |
+
"accepted": 109,
|
| 581 |
+
"coverage": 0.9396551724137931,
|
| 582 |
+
"errorRate": 0.009174311926605505
|
| 583 |
+
},
|
| 584 |
+
{
|
| 585 |
+
"threshold": 0.9,
|
| 586 |
+
"accepted": 107,
|
| 587 |
+
"coverage": 0.9224137931034483,
|
| 588 |
+
"errorRate": 0.0
|
| 589 |
+
},
|
| 590 |
+
{
|
| 591 |
+
"threshold": 0.95,
|
| 592 |
+
"accepted": 107,
|
| 593 |
+
"coverage": 0.9224137931034483,
|
| 594 |
+
"errorRate": 0.0
|
| 595 |
+
},
|
| 596 |
+
{
|
| 597 |
+
"threshold": 0.99,
|
| 598 |
+
"accepted": 101,
|
| 599 |
+
"coverage": 0.8706896551724138,
|
| 600 |
+
"errorRate": 0.0
|
| 601 |
+
}
|
| 602 |
+
],
|
| 603 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 604 |
+
},
|
| 605 |
+
"tweet_offensive": {
|
| 606 |
+
"questions": 80,
|
| 607 |
+
"answered": 80,
|
| 608 |
+
"errors": 0,
|
| 609 |
+
"correct": 65,
|
| 610 |
+
"accuracyAllQuestions": 0.8125,
|
| 611 |
+
"brierAnswered": 0.265965,
|
| 612 |
+
"nllAnswered": 0.4124258192766881,
|
| 613 |
+
"ece10Answered": 0.02999999999999997,
|
| 614 |
+
"confidentWrongAt09": 2,
|
| 615 |
+
"coverageByFixedThreshold": [
|
| 616 |
+
{
|
| 617 |
+
"threshold": 0.5,
|
| 618 |
+
"accepted": 80,
|
| 619 |
+
"coverage": 1.0,
|
| 620 |
+
"errorRate": 0.1875
|
| 621 |
+
},
|
| 622 |
+
{
|
| 623 |
+
"threshold": 0.7,
|
| 624 |
+
"accepted": 56,
|
| 625 |
+
"coverage": 0.7,
|
| 626 |
+
"errorRate": 0.10714285714285714
|
| 627 |
+
},
|
| 628 |
+
{
|
| 629 |
+
"threshold": 0.8,
|
| 630 |
+
"accepted": 44,
|
| 631 |
+
"coverage": 0.55,
|
| 632 |
+
"errorRate": 0.06818181818181818
|
| 633 |
+
},
|
| 634 |
+
{
|
| 635 |
+
"threshold": 0.9,
|
| 636 |
+
"accepted": 30,
|
| 637 |
+
"coverage": 0.375,
|
| 638 |
+
"errorRate": 0.06666666666666667
|
| 639 |
+
},
|
| 640 |
+
{
|
| 641 |
+
"threshold": 0.95,
|
| 642 |
+
"accepted": 17,
|
| 643 |
+
"coverage": 0.2125,
|
| 644 |
+
"errorRate": 0.0
|
| 645 |
+
},
|
| 646 |
+
{
|
| 647 |
+
"threshold": 0.99,
|
| 648 |
+
"accepted": 1,
|
| 649 |
+
"coverage": 0.0125,
|
| 650 |
+
"errorRate": 0.0
|
| 651 |
+
}
|
| 652 |
+
],
|
| 653 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 654 |
+
}
|
| 655 |
+
},
|
| 656 |
+
"provenance": {
|
| 657 |
+
"model": "jev-1.14.0",
|
| 658 |
+
"endpoint": "https://api.typesafe.ai/v1/systemone",
|
| 659 |
+
"records": 812,
|
| 660 |
+
"datasetSha256": "9266d6ea1ce83c9755a60cf48c7469b0fa3264c229e96f4142d683bada8fef91",
|
| 661 |
+
"estimatedCapUsd": 0.15,
|
| 662 |
+
"conservativeByteTokenUpperEstimateUsd": 0.018575508,
|
| 663 |
+
"priceUsdPerMillionInputTokens": 0.042,
|
| 664 |
+
"noAutomaticRetries": true,
|
| 665 |
+
"publicAuthoredInputsOnly": true
|
| 666 |
+
},
|
| 667 |
+
"servedModels": [
|
| 668 |
+
"jev-1.14.0"
|
| 669 |
+
],
|
| 670 |
+
"records": 812,
|
| 671 |
+
"usageInputTokens": 323757,
|
| 672 |
+
"estimatedApiCostUsd": 0.013597794000000002,
|
| 673 |
+
"seconds": 289.84684014320374,
|
| 674 |
+
"wallLatencyMs": {
|
| 675 |
+
"median": 311.6002500173636
|
| 676 |
+
},
|
| 677 |
+
"latencyMeaning": "Remote network/client wall time; excludes reused preflight; not comparable GPU kernels or throughput"
|
| 678 |
+
}
|
evaluation/benchmarks/jev/rows.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
evaluation/benchmarks/kev/predictions.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
evaluation/benchmarks/kev/report.json
ADDED
|
@@ -0,0 +1,675 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"overall": {
|
| 3 |
+
"questions": 908,
|
| 4 |
+
"answered": 908,
|
| 5 |
+
"errors": 0,
|
| 6 |
+
"correct": 756,
|
| 7 |
+
"accuracyAllQuestions": 0.8325991189427313,
|
| 8 |
+
"brierAnswered": 0.23049057438945586,
|
| 9 |
+
"nllAnswered": 0.42905102574468273,
|
| 10 |
+
"ece10Answered": 0.051600521203983195,
|
| 11 |
+
"confidentWrongAt09": 6,
|
| 12 |
+
"coverageByFixedThreshold": [
|
| 13 |
+
{
|
| 14 |
+
"threshold": 0.5,
|
| 15 |
+
"accepted": 828,
|
| 16 |
+
"coverage": 0.9118942731277533,
|
| 17 |
+
"errorRate": 0.12560386473429952
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"threshold": 0.7,
|
| 21 |
+
"accepted": 667,
|
| 22 |
+
"coverage": 0.7345814977973568,
|
| 23 |
+
"errorRate": 0.05997001499250375
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"threshold": 0.8,
|
| 27 |
+
"accepted": 531,
|
| 28 |
+
"coverage": 0.5848017621145375,
|
| 29 |
+
"errorRate": 0.04331450094161959
|
| 30 |
+
},
|
| 31 |
+
{
|
| 32 |
+
"threshold": 0.9,
|
| 33 |
+
"accepted": 368,
|
| 34 |
+
"coverage": 0.4052863436123348,
|
| 35 |
+
"errorRate": 0.016304347826086956
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"threshold": 0.95,
|
| 39 |
+
"accepted": 276,
|
| 40 |
+
"coverage": 0.3039647577092511,
|
| 41 |
+
"errorRate": 0.007246376811594203
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"threshold": 0.99,
|
| 45 |
+
"accepted": 88,
|
| 46 |
+
"coverage": 0.09691629955947137,
|
| 47 |
+
"errorRate": 0.0
|
| 48 |
+
}
|
| 49 |
+
],
|
| 50 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 51 |
+
},
|
| 52 |
+
"byType": {
|
| 53 |
+
"choice": {
|
| 54 |
+
"questions": 492,
|
| 55 |
+
"answered": 492,
|
| 56 |
+
"errors": 0,
|
| 57 |
+
"correct": 400,
|
| 58 |
+
"accuracyAllQuestions": 0.8130081300813008,
|
| 59 |
+
"brierAnswered": 0.24971593029461644,
|
| 60 |
+
"nllAnswered": 0.5001996781390384,
|
| 61 |
+
"ece10Answered": 0.05950644744126973,
|
| 62 |
+
"confidentWrongAt09": 2,
|
| 63 |
+
"coverageByFixedThreshold": [
|
| 64 |
+
{
|
| 65 |
+
"threshold": 0.5,
|
| 66 |
+
"accepted": 421,
|
| 67 |
+
"coverage": 0.8556910569105691,
|
| 68 |
+
"errorRate": 0.11163895486935867
|
| 69 |
+
},
|
| 70 |
+
{
|
| 71 |
+
"threshold": 0.7,
|
| 72 |
+
"accepted": 345,
|
| 73 |
+
"coverage": 0.7012195121951219,
|
| 74 |
+
"errorRate": 0.04927536231884058
|
| 75 |
+
},
|
| 76 |
+
{
|
| 77 |
+
"threshold": 0.8,
|
| 78 |
+
"accepted": 276,
|
| 79 |
+
"coverage": 0.5609756097560976,
|
| 80 |
+
"errorRate": 0.036231884057971016
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"threshold": 0.9,
|
| 84 |
+
"accepted": 193,
|
| 85 |
+
"coverage": 0.39227642276422764,
|
| 86 |
+
"errorRate": 0.010362694300518135
|
| 87 |
+
},
|
| 88 |
+
{
|
| 89 |
+
"threshold": 0.95,
|
| 90 |
+
"accepted": 144,
|
| 91 |
+
"coverage": 0.2926829268292683,
|
| 92 |
+
"errorRate": 0.0
|
| 93 |
+
},
|
| 94 |
+
{
|
| 95 |
+
"threshold": 0.99,
|
| 96 |
+
"accepted": 36,
|
| 97 |
+
"coverage": 0.07317073170731707,
|
| 98 |
+
"errorRate": 0.0
|
| 99 |
+
}
|
| 100 |
+
],
|
| 101 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 102 |
+
},
|
| 103 |
+
"noul": {
|
| 104 |
+
"questions": 328,
|
| 105 |
+
"answered": 328,
|
| 106 |
+
"errors": 0,
|
| 107 |
+
"correct": 284,
|
| 108 |
+
"accuracyAllQuestions": 0.8658536585365854,
|
| 109 |
+
"brierAnswered": 0.19400091681333753,
|
| 110 |
+
"nllAnswered": 0.3162898738046143,
|
| 111 |
+
"ece10Answered": 0.03553192036633847,
|
| 112 |
+
"confidentWrongAt09": 3,
|
| 113 |
+
"coverageByFixedThreshold": [
|
| 114 |
+
{
|
| 115 |
+
"threshold": 0.5,
|
| 116 |
+
"accepted": 328,
|
| 117 |
+
"coverage": 1.0,
|
| 118 |
+
"errorRate": 0.13414634146341464
|
| 119 |
+
},
|
| 120 |
+
{
|
| 121 |
+
"threshold": 0.7,
|
| 122 |
+
"accepted": 264,
|
| 123 |
+
"coverage": 0.8048780487804879,
|
| 124 |
+
"errorRate": 0.07196969696969698
|
| 125 |
+
},
|
| 126 |
+
{
|
| 127 |
+
"threshold": 0.8,
|
| 128 |
+
"accepted": 216,
|
| 129 |
+
"coverage": 0.6585365853658537,
|
| 130 |
+
"errorRate": 0.05092592592592592
|
| 131 |
+
},
|
| 132 |
+
{
|
| 133 |
+
"threshold": 0.9,
|
| 134 |
+
"accepted": 154,
|
| 135 |
+
"coverage": 0.4695121951219512,
|
| 136 |
+
"errorRate": 0.01948051948051948
|
| 137 |
+
},
|
| 138 |
+
{
|
| 139 |
+
"threshold": 0.95,
|
| 140 |
+
"accepted": 119,
|
| 141 |
+
"coverage": 0.3628048780487805,
|
| 142 |
+
"errorRate": 0.01680672268907563
|
| 143 |
+
},
|
| 144 |
+
{
|
| 145 |
+
"threshold": 0.99,
|
| 146 |
+
"accepted": 50,
|
| 147 |
+
"coverage": 0.1524390243902439,
|
| 148 |
+
"errorRate": 0.0
|
| 149 |
+
}
|
| 150 |
+
],
|
| 151 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 152 |
+
},
|
| 153 |
+
"score": {
|
| 154 |
+
"questions": 88,
|
| 155 |
+
"answered": 88,
|
| 156 |
+
"errors": 0,
|
| 157 |
+
"correct": 72,
|
| 158 |
+
"accuracyAllQuestions": 0.8181818181818182,
|
| 159 |
+
"brierAnswered": 0.2590102627943176,
|
| 160 |
+
"nllAnswered": 0.451556944589222,
|
| 161 |
+
"ece10Answered": 0.08293436635062447,
|
| 162 |
+
"confidentWrongAt09": 1,
|
| 163 |
+
"coverageByFixedThreshold": [
|
| 164 |
+
{
|
| 165 |
+
"threshold": 0.5,
|
| 166 |
+
"accepted": 79,
|
| 167 |
+
"coverage": 0.8977272727272727,
|
| 168 |
+
"errorRate": 0.16455696202531644
|
| 169 |
+
},
|
| 170 |
+
{
|
| 171 |
+
"threshold": 0.7,
|
| 172 |
+
"accepted": 58,
|
| 173 |
+
"coverage": 0.6590909090909091,
|
| 174 |
+
"errorRate": 0.06896551724137931
|
| 175 |
+
},
|
| 176 |
+
{
|
| 177 |
+
"threshold": 0.8,
|
| 178 |
+
"accepted": 39,
|
| 179 |
+
"coverage": 0.4431818181818182,
|
| 180 |
+
"errorRate": 0.05128205128205128
|
| 181 |
+
},
|
| 182 |
+
{
|
| 183 |
+
"threshold": 0.9,
|
| 184 |
+
"accepted": 21,
|
| 185 |
+
"coverage": 0.23863636363636365,
|
| 186 |
+
"errorRate": 0.047619047619047616
|
| 187 |
+
},
|
| 188 |
+
{
|
| 189 |
+
"threshold": 0.95,
|
| 190 |
+
"accepted": 13,
|
| 191 |
+
"coverage": 0.14772727272727273,
|
| 192 |
+
"errorRate": 0.0
|
| 193 |
+
},
|
| 194 |
+
{
|
| 195 |
+
"threshold": 0.99,
|
| 196 |
+
"accepted": 2,
|
| 197 |
+
"coverage": 0.022727272727272728,
|
| 198 |
+
"errorRate": 0.0
|
| 199 |
+
}
|
| 200 |
+
],
|
| 201 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 202 |
+
}
|
| 203 |
+
},
|
| 204 |
+
"bySource": {
|
| 205 |
+
"authored-russian-typed": {
|
| 206 |
+
"questions": 144,
|
| 207 |
+
"answered": 144,
|
| 208 |
+
"errors": 0,
|
| 209 |
+
"correct": 143,
|
| 210 |
+
"accuracyAllQuestions": 0.9930555555555556,
|
| 211 |
+
"brierAnswered": 0.10416501037063129,
|
| 212 |
+
"nllAnswered": 0.2628194880491901,
|
| 213 |
+
"ece10Answered": 0.21249311017124553,
|
| 214 |
+
"confidentWrongAt09": 0,
|
| 215 |
+
"coverageByFixedThreshold": [
|
| 216 |
+
{
|
| 217 |
+
"threshold": 0.5,
|
| 218 |
+
"accepted": 137,
|
| 219 |
+
"coverage": 0.9513888888888888,
|
| 220 |
+
"errorRate": 0.0
|
| 221 |
+
},
|
| 222 |
+
{
|
| 223 |
+
"threshold": 0.7,
|
| 224 |
+
"accepted": 125,
|
| 225 |
+
"coverage": 0.8680555555555556,
|
| 226 |
+
"errorRate": 0.0
|
| 227 |
+
},
|
| 228 |
+
{
|
| 229 |
+
"threshold": 0.8,
|
| 230 |
+
"accepted": 63,
|
| 231 |
+
"coverage": 0.4375,
|
| 232 |
+
"errorRate": 0.0
|
| 233 |
+
},
|
| 234 |
+
{
|
| 235 |
+
"threshold": 0.9,
|
| 236 |
+
"accepted": 18,
|
| 237 |
+
"coverage": 0.125,
|
| 238 |
+
"errorRate": 0.0
|
| 239 |
+
},
|
| 240 |
+
{
|
| 241 |
+
"threshold": 0.95,
|
| 242 |
+
"accepted": 14,
|
| 243 |
+
"coverage": 0.09722222222222222,
|
| 244 |
+
"errorRate": 0.0
|
| 245 |
+
},
|
| 246 |
+
{
|
| 247 |
+
"threshold": 0.99,
|
| 248 |
+
"accepted": 1,
|
| 249 |
+
"coverage": 0.006944444444444444,
|
| 250 |
+
"errorRate": 0.0
|
| 251 |
+
}
|
| 252 |
+
],
|
| 253 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 254 |
+
},
|
| 255 |
+
"composition_holdout": {
|
| 256 |
+
"questions": 96,
|
| 257 |
+
"answered": 96,
|
| 258 |
+
"errors": 0,
|
| 259 |
+
"correct": 94,
|
| 260 |
+
"accuracyAllQuestions": 0.9791666666666666,
|
| 261 |
+
"brierAnswered": 0.07265750882301934,
|
| 262 |
+
"nllAnswered": 0.14970972837197358,
|
| 263 |
+
"ece10Answered": 0.07865301631331481,
|
| 264 |
+
"confidentWrongAt09": 1,
|
| 265 |
+
"coverageByFixedThreshold": [
|
| 266 |
+
{
|
| 267 |
+
"threshold": 0.5,
|
| 268 |
+
"accepted": 96,
|
| 269 |
+
"coverage": 1.0,
|
| 270 |
+
"errorRate": 0.020833333333333332
|
| 271 |
+
},
|
| 272 |
+
{
|
| 273 |
+
"threshold": 0.7,
|
| 274 |
+
"accepted": 85,
|
| 275 |
+
"coverage": 0.8854166666666666,
|
| 276 |
+
"errorRate": 0.011764705882352941
|
| 277 |
+
},
|
| 278 |
+
{
|
| 279 |
+
"threshold": 0.8,
|
| 280 |
+
"accepted": 80,
|
| 281 |
+
"coverage": 0.8333333333333334,
|
| 282 |
+
"errorRate": 0.0125
|
| 283 |
+
},
|
| 284 |
+
{
|
| 285 |
+
"threshold": 0.9,
|
| 286 |
+
"accepted": 64,
|
| 287 |
+
"coverage": 0.6666666666666666,
|
| 288 |
+
"errorRate": 0.015625
|
| 289 |
+
},
|
| 290 |
+
{
|
| 291 |
+
"threshold": 0.95,
|
| 292 |
+
"accepted": 52,
|
| 293 |
+
"coverage": 0.5416666666666666,
|
| 294 |
+
"errorRate": 0.0
|
| 295 |
+
},
|
| 296 |
+
{
|
| 297 |
+
"threshold": 0.99,
|
| 298 |
+
"accepted": 13,
|
| 299 |
+
"coverage": 0.13541666666666666,
|
| 300 |
+
"errorRate": 0.0
|
| 301 |
+
}
|
| 302 |
+
],
|
| 303 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 304 |
+
},
|
| 305 |
+
"emotion": {
|
| 306 |
+
"questions": 116,
|
| 307 |
+
"answered": 116,
|
| 308 |
+
"errors": 0,
|
| 309 |
+
"correct": 65,
|
| 310 |
+
"accuracyAllQuestions": 0.5603448275862069,
|
| 311 |
+
"brierAnswered": 0.5283768574785999,
|
| 312 |
+
"nllAnswered": 1.0464486013956598,
|
| 313 |
+
"ece10Answered": 0.10797638647123994,
|
| 314 |
+
"confidentWrongAt09": 0,
|
| 315 |
+
"coverageByFixedThreshold": [
|
| 316 |
+
{
|
| 317 |
+
"threshold": 0.5,
|
| 318 |
+
"accepted": 89,
|
| 319 |
+
"coverage": 0.7672413793103449,
|
| 320 |
+
"errorRate": 0.33707865168539325
|
| 321 |
+
},
|
| 322 |
+
{
|
| 323 |
+
"threshold": 0.7,
|
| 324 |
+
"accepted": 48,
|
| 325 |
+
"coverage": 0.41379310344827586,
|
| 326 |
+
"errorRate": 0.20833333333333334
|
| 327 |
+
},
|
| 328 |
+
{
|
| 329 |
+
"threshold": 0.8,
|
| 330 |
+
"accepted": 34,
|
| 331 |
+
"coverage": 0.29310344827586204,
|
| 332 |
+
"errorRate": 0.17647058823529413
|
| 333 |
+
},
|
| 334 |
+
{
|
| 335 |
+
"threshold": 0.9,
|
| 336 |
+
"accepted": 12,
|
| 337 |
+
"coverage": 0.10344827586206896,
|
| 338 |
+
"errorRate": 0.0
|
| 339 |
+
},
|
| 340 |
+
{
|
| 341 |
+
"threshold": 0.95,
|
| 342 |
+
"accepted": 3,
|
| 343 |
+
"coverage": 0.02586206896551724,
|
| 344 |
+
"errorRate": 0.0
|
| 345 |
+
},
|
| 346 |
+
{
|
| 347 |
+
"threshold": 0.99,
|
| 348 |
+
"accepted": 0,
|
| 349 |
+
"coverage": 0.0,
|
| 350 |
+
"errorRate": null
|
| 351 |
+
}
|
| 352 |
+
],
|
| 353 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 354 |
+
},
|
| 355 |
+
"legacy_holdout": {
|
| 356 |
+
"questions": 80,
|
| 357 |
+
"answered": 80,
|
| 358 |
+
"errors": 0,
|
| 359 |
+
"correct": 65,
|
| 360 |
+
"accuracyAllQuestions": 0.8125,
|
| 361 |
+
"brierAnswered": 0.18524784583046267,
|
| 362 |
+
"nllAnswered": 0.28116834855200057,
|
| 363 |
+
"ece10Answered": 0.08460090352211151,
|
| 364 |
+
"confidentWrongAt09": 1,
|
| 365 |
+
"coverageByFixedThreshold": [
|
| 366 |
+
{
|
| 367 |
+
"threshold": 0.5,
|
| 368 |
+
"accepted": 78,
|
| 369 |
+
"coverage": 0.975,
|
| 370 |
+
"errorRate": 0.16666666666666666
|
| 371 |
+
},
|
| 372 |
+
{
|
| 373 |
+
"threshold": 0.7,
|
| 374 |
+
"accepted": 66,
|
| 375 |
+
"coverage": 0.825,
|
| 376 |
+
"errorRate": 0.06060606060606061
|
| 377 |
+
},
|
| 378 |
+
{
|
| 379 |
+
"threshold": 0.8,
|
| 380 |
+
"accepted": 63,
|
| 381 |
+
"coverage": 0.7875,
|
| 382 |
+
"errorRate": 0.031746031746031744
|
| 383 |
+
},
|
| 384 |
+
{
|
| 385 |
+
"threshold": 0.9,
|
| 386 |
+
"accepted": 58,
|
| 387 |
+
"coverage": 0.725,
|
| 388 |
+
"errorRate": 0.017241379310344827
|
| 389 |
+
},
|
| 390 |
+
{
|
| 391 |
+
"threshold": 0.95,
|
| 392 |
+
"accepted": 53,
|
| 393 |
+
"coverage": 0.6625,
|
| 394 |
+
"errorRate": 0.0
|
| 395 |
+
},
|
| 396 |
+
{
|
| 397 |
+
"threshold": 0.99,
|
| 398 |
+
"accepted": 39,
|
| 399 |
+
"coverage": 0.4875,
|
| 400 |
+
"errorRate": 0.0
|
| 401 |
+
}
|
| 402 |
+
],
|
| 403 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 404 |
+
},
|
| 405 |
+
"mmlu": {
|
| 406 |
+
"questions": 116,
|
| 407 |
+
"answered": 116,
|
| 408 |
+
"errors": 0,
|
| 409 |
+
"correct": 79,
|
| 410 |
+
"accuracyAllQuestions": 0.6810344827586207,
|
| 411 |
+
"brierAnswered": 0.3964835619347099,
|
| 412 |
+
"nllAnswered": 0.740201823820986,
|
| 413 |
+
"ece10Answered": 0.06660618296076969,
|
| 414 |
+
"confidentWrongAt09": 1,
|
| 415 |
+
"coverageByFixedThreshold": [
|
| 416 |
+
{
|
| 417 |
+
"threshold": 0.5,
|
| 418 |
+
"accepted": 74,
|
| 419 |
+
"coverage": 0.6379310344827587,
|
| 420 |
+
"errorRate": 0.17567567567567569
|
| 421 |
+
},
|
| 422 |
+
{
|
| 423 |
+
"threshold": 0.7,
|
| 424 |
+
"accepted": 59,
|
| 425 |
+
"coverage": 0.5086206896551724,
|
| 426 |
+
"errorRate": 0.1016949152542373
|
| 427 |
+
},
|
| 428 |
+
{
|
| 429 |
+
"threshold": 0.8,
|
| 430 |
+
"accepted": 44,
|
| 431 |
+
"coverage": 0.3793103448275862,
|
| 432 |
+
"errorRate": 0.06818181818181818
|
| 433 |
+
},
|
| 434 |
+
{
|
| 435 |
+
"threshold": 0.9,
|
| 436 |
+
"accepted": 28,
|
| 437 |
+
"coverage": 0.2413793103448276,
|
| 438 |
+
"errorRate": 0.03571428571428571
|
| 439 |
+
},
|
| 440 |
+
{
|
| 441 |
+
"threshold": 0.95,
|
| 442 |
+
"accepted": 15,
|
| 443 |
+
"coverage": 0.12931034482758622,
|
| 444 |
+
"errorRate": 0.0
|
| 445 |
+
},
|
| 446 |
+
{
|
| 447 |
+
"threshold": 0.99,
|
| 448 |
+
"accepted": 1,
|
| 449 |
+
"coverage": 0.008620689655172414,
|
| 450 |
+
"errorRate": 0.0
|
| 451 |
+
}
|
| 452 |
+
],
|
| 453 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 454 |
+
},
|
| 455 |
+
"paws": {
|
| 456 |
+
"questions": 80,
|
| 457 |
+
"answered": 80,
|
| 458 |
+
"errors": 0,
|
| 459 |
+
"correct": 64,
|
| 460 |
+
"accuracyAllQuestions": 0.8,
|
| 461 |
+
"brierAnswered": 0.30918811576472116,
|
| 462 |
+
"nllAnswered": 0.48629027291665167,
|
| 463 |
+
"ece10Answered": 0.0669704500818625,
|
| 464 |
+
"confidentWrongAt09": 2,
|
| 465 |
+
"coverageByFixedThreshold": [
|
| 466 |
+
{
|
| 467 |
+
"threshold": 0.5,
|
| 468 |
+
"accepted": 80,
|
| 469 |
+
"coverage": 1.0,
|
| 470 |
+
"errorRate": 0.2
|
| 471 |
+
},
|
| 472 |
+
{
|
| 473 |
+
"threshold": 0.7,
|
| 474 |
+
"accepted": 65,
|
| 475 |
+
"coverage": 0.8125,
|
| 476 |
+
"errorRate": 0.16923076923076924
|
| 477 |
+
},
|
| 478 |
+
{
|
| 479 |
+
"threshold": 0.8,
|
| 480 |
+
"accepted": 55,
|
| 481 |
+
"coverage": 0.6875,
|
| 482 |
+
"errorRate": 0.12727272727272726
|
| 483 |
+
},
|
| 484 |
+
{
|
| 485 |
+
"threshold": 0.9,
|
| 486 |
+
"accepted": 37,
|
| 487 |
+
"coverage": 0.4625,
|
| 488 |
+
"errorRate": 0.05405405405405406
|
| 489 |
+
},
|
| 490 |
+
{
|
| 491 |
+
"threshold": 0.95,
|
| 492 |
+
"accepted": 23,
|
| 493 |
+
"coverage": 0.2875,
|
| 494 |
+
"errorRate": 0.08695652173913043
|
| 495 |
+
},
|
| 496 |
+
{
|
| 497 |
+
"threshold": 0.99,
|
| 498 |
+
"accepted": 2,
|
| 499 |
+
"coverage": 0.025,
|
| 500 |
+
"errorRate": 0.0
|
| 501 |
+
}
|
| 502 |
+
],
|
| 503 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 504 |
+
},
|
| 505 |
+
"qnli": {
|
| 506 |
+
"questions": 80,
|
| 507 |
+
"answered": 80,
|
| 508 |
+
"errors": 0,
|
| 509 |
+
"correct": 73,
|
| 510 |
+
"accuracyAllQuestions": 0.9125,
|
| 511 |
+
"brierAnswered": 0.14017388395851296,
|
| 512 |
+
"nllAnswered": 0.2368256396206597,
|
| 513 |
+
"ece10Answered": 0.028304750133247607,
|
| 514 |
+
"confidentWrongAt09": 1,
|
| 515 |
+
"coverageByFixedThreshold": [
|
| 516 |
+
{
|
| 517 |
+
"threshold": 0.5,
|
| 518 |
+
"accepted": 80,
|
| 519 |
+
"coverage": 1.0,
|
| 520 |
+
"errorRate": 0.0875
|
| 521 |
+
},
|
| 522 |
+
{
|
| 523 |
+
"threshold": 0.7,
|
| 524 |
+
"accepted": 70,
|
| 525 |
+
"coverage": 0.875,
|
| 526 |
+
"errorRate": 0.05714285714285714
|
| 527 |
+
},
|
| 528 |
+
{
|
| 529 |
+
"threshold": 0.8,
|
| 530 |
+
"accepted": 63,
|
| 531 |
+
"coverage": 0.7875,
|
| 532 |
+
"errorRate": 0.031746031746031744
|
| 533 |
+
},
|
| 534 |
+
{
|
| 535 |
+
"threshold": 0.9,
|
| 536 |
+
"accepted": 50,
|
| 537 |
+
"coverage": 0.625,
|
| 538 |
+
"errorRate": 0.02
|
| 539 |
+
},
|
| 540 |
+
{
|
| 541 |
+
"threshold": 0.95,
|
| 542 |
+
"accepted": 38,
|
| 543 |
+
"coverage": 0.475,
|
| 544 |
+
"errorRate": 0.0
|
| 545 |
+
},
|
| 546 |
+
{
|
| 547 |
+
"threshold": 0.99,
|
| 548 |
+
"accepted": 10,
|
| 549 |
+
"coverage": 0.125,
|
| 550 |
+
"errorRate": 0.0
|
| 551 |
+
}
|
| 552 |
+
],
|
| 553 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 554 |
+
},
|
| 555 |
+
"sciq": {
|
| 556 |
+
"questions": 116,
|
| 557 |
+
"answered": 116,
|
| 558 |
+
"errors": 0,
|
| 559 |
+
"correct": 114,
|
| 560 |
+
"accuracyAllQuestions": 0.9827586206896551,
|
| 561 |
+
"brierAnswered": 0.04432013428162752,
|
| 562 |
+
"nllAnswered": 0.11124410753498573,
|
| 563 |
+
"ece10Answered": 0.07043185982245576,
|
| 564 |
+
"confidentWrongAt09": 0,
|
| 565 |
+
"coverageByFixedThreshold": [
|
| 566 |
+
{
|
| 567 |
+
"threshold": 0.5,
|
| 568 |
+
"accepted": 114,
|
| 569 |
+
"coverage": 0.9827586206896551,
|
| 570 |
+
"errorRate": 0.017543859649122806
|
| 571 |
+
},
|
| 572 |
+
{
|
| 573 |
+
"threshold": 0.7,
|
| 574 |
+
"accepted": 106,
|
| 575 |
+
"coverage": 0.9137931034482759,
|
| 576 |
+
"errorRate": 0.0
|
| 577 |
+
},
|
| 578 |
+
{
|
| 579 |
+
"threshold": 0.8,
|
| 580 |
+
"accepted": 98,
|
| 581 |
+
"coverage": 0.8448275862068966,
|
| 582 |
+
"errorRate": 0.0
|
| 583 |
+
},
|
| 584 |
+
{
|
| 585 |
+
"threshold": 0.9,
|
| 586 |
+
"accepted": 89,
|
| 587 |
+
"coverage": 0.7672413793103449,
|
| 588 |
+
"errorRate": 0.0
|
| 589 |
+
},
|
| 590 |
+
{
|
| 591 |
+
"threshold": 0.95,
|
| 592 |
+
"accepted": 74,
|
| 593 |
+
"coverage": 0.6379310344827587,
|
| 594 |
+
"errorRate": 0.0
|
| 595 |
+
},
|
| 596 |
+
{
|
| 597 |
+
"threshold": 0.99,
|
| 598 |
+
"accepted": 22,
|
| 599 |
+
"coverage": 0.1896551724137931,
|
| 600 |
+
"errorRate": 0.0
|
| 601 |
+
}
|
| 602 |
+
],
|
| 603 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 604 |
+
},
|
| 605 |
+
"tweet_offensive": {
|
| 606 |
+
"questions": 80,
|
| 607 |
+
"answered": 80,
|
| 608 |
+
"errors": 0,
|
| 609 |
+
"correct": 59,
|
| 610 |
+
"accuracyAllQuestions": 0.7375,
|
| 611 |
+
"brierAnswered": 0.3014603416542088,
|
| 612 |
+
"nllAnswered": 0.4607710560880617,
|
| 613 |
+
"ece10Answered": 0.0959448419045657,
|
| 614 |
+
"confidentWrongAt09": 0,
|
| 615 |
+
"coverageByFixedThreshold": [
|
| 616 |
+
{
|
| 617 |
+
"threshold": 0.5,
|
| 618 |
+
"accepted": 80,
|
| 619 |
+
"coverage": 1.0,
|
| 620 |
+
"errorRate": 0.2625
|
| 621 |
+
},
|
| 622 |
+
{
|
| 623 |
+
"threshold": 0.7,
|
| 624 |
+
"accepted": 43,
|
| 625 |
+
"coverage": 0.5375,
|
| 626 |
+
"errorRate": 0.09302325581395349
|
| 627 |
+
},
|
| 628 |
+
{
|
| 629 |
+
"threshold": 0.8,
|
| 630 |
+
"accepted": 31,
|
| 631 |
+
"coverage": 0.3875,
|
| 632 |
+
"errorRate": 0.06451612903225806
|
| 633 |
+
},
|
| 634 |
+
{
|
| 635 |
+
"threshold": 0.9,
|
| 636 |
+
"accepted": 12,
|
| 637 |
+
"coverage": 0.15,
|
| 638 |
+
"errorRate": 0.0
|
| 639 |
+
},
|
| 640 |
+
{
|
| 641 |
+
"threshold": 0.95,
|
| 642 |
+
"accepted": 4,
|
| 643 |
+
"coverage": 0.05,
|
| 644 |
+
"errorRate": 0.0
|
| 645 |
+
},
|
| 646 |
+
{
|
| 647 |
+
"threshold": 0.99,
|
| 648 |
+
"accepted": 0,
|
| 649 |
+
"coverage": 0.0,
|
| 650 |
+
"errorRate": null
|
| 651 |
+
}
|
| 652 |
+
],
|
| 653 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 654 |
+
}
|
| 655 |
+
},
|
| 656 |
+
"provenance": {
|
| 657 |
+
"revision": "6cfce5c2fa4b4bd64026336ab649c5ca78857d52",
|
| 658 |
+
"codeRevision": "2074cc44256200bb37540103fa4572901faa10f3",
|
| 659 |
+
"dataSha256": "9266d6ea1ce83c9755a60cf48c7469b0fa3264c229e96f4142d683bada8fef91",
|
| 660 |
+
"precision": "BF16 merged, official eager SDPA; no graphs/fused",
|
| 661 |
+
"gpu": "NVIDIA A10G",
|
| 662 |
+
"torch": "2.8.0+cu128",
|
| 663 |
+
"temperature": 2.406050072164233
|
| 664 |
+
},
|
| 665 |
+
"servedModels": [
|
| 666 |
+
"kev-4b@6cfce5c2fa4b4bd64026336ab649c5ca78857d52"
|
| 667 |
+
],
|
| 668 |
+
"records": 812,
|
| 669 |
+
"startedUtc": null,
|
| 670 |
+
"wallLatencyMs": {
|
| 671 |
+
"median": 67.72997199999509,
|
| 672 |
+
"p95": 83.42849299998534
|
| 673 |
+
},
|
| 674 |
+
"latencyMeaning": "Sequential wall time including client overhead/cold calls; not model-only time or throughput"
|
| 675 |
+
}
|
evaluation/benchmarks/kev/rows.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
evaluation/benchmarks/results.json
ADDED
|
@@ -0,0 +1,2334 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"models": {
|
| 3 |
+
"Fev": {
|
| 4 |
+
"all": {
|
| 5 |
+
"questions": 908,
|
| 6 |
+
"answered": 908,
|
| 7 |
+
"errors": 0,
|
| 8 |
+
"correct": 783,
|
| 9 |
+
"accuracyAllQuestions": 0.8623348017621145,
|
| 10 |
+
"brierAnswered": 0.19616500539754675,
|
| 11 |
+
"nllAnswered": 0.374025080065555,
|
| 12 |
+
"ece10Answered": 0.04207503699664424,
|
| 13 |
+
"confidentWrongAt09": 26,
|
| 14 |
+
"coverageByFixedThreshold": [
|
| 15 |
+
{
|
| 16 |
+
"threshold": 0.5,
|
| 17 |
+
"accepted": 889,
|
| 18 |
+
"coverage": 0.9790748898678414,
|
| 19 |
+
"errorRate": 0.12373453318335208
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"threshold": 0.7,
|
| 23 |
+
"accepted": 819,
|
| 24 |
+
"coverage": 0.9019823788546255,
|
| 25 |
+
"errorRate": 0.08424908424908426
|
| 26 |
+
},
|
| 27 |
+
{
|
| 28 |
+
"threshold": 0.8,
|
| 29 |
+
"accepted": 767,
|
| 30 |
+
"coverage": 0.8447136563876652,
|
| 31 |
+
"errorRate": 0.06649282920469361
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"threshold": 0.9,
|
| 35 |
+
"accepted": 676,
|
| 36 |
+
"coverage": 0.7444933920704846,
|
| 37 |
+
"errorRate": 0.038461538461538464
|
| 38 |
+
},
|
| 39 |
+
{
|
| 40 |
+
"threshold": 0.95,
|
| 41 |
+
"accepted": 567,
|
| 42 |
+
"coverage": 0.6244493392070485,
|
| 43 |
+
"errorRate": 0.012345679012345678
|
| 44 |
+
},
|
| 45 |
+
{
|
| 46 |
+
"threshold": 0.99,
|
| 47 |
+
"accepted": 0,
|
| 48 |
+
"coverage": 0.0,
|
| 49 |
+
"errorRate": null
|
| 50 |
+
}
|
| 51 |
+
],
|
| 52 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here.",
|
| 53 |
+
"ordinalMeanAbsoluteError": 0.060723818086750196
|
| 54 |
+
},
|
| 55 |
+
"publicTransfer": {
|
| 56 |
+
"questions": 764,
|
| 57 |
+
"answered": 764,
|
| 58 |
+
"errors": 0,
|
| 59 |
+
"correct": 639,
|
| 60 |
+
"accuracyAllQuestions": 0.8363874345549738,
|
| 61 |
+
"brierAnswered": 0.2329500652363337,
|
| 62 |
+
"nllAnswered": 0.43991052107250256,
|
| 63 |
+
"ece10Answered": 0.054555210421981894,
|
| 64 |
+
"confidentWrongAt09": 26,
|
| 65 |
+
"coverageByFixedThreshold": [
|
| 66 |
+
{
|
| 67 |
+
"threshold": 0.5,
|
| 68 |
+
"accepted": 745,
|
| 69 |
+
"coverage": 0.975130890052356,
|
| 70 |
+
"errorRate": 0.1476510067114094
|
| 71 |
+
},
|
| 72 |
+
{
|
| 73 |
+
"threshold": 0.7,
|
| 74 |
+
"accepted": 675,
|
| 75 |
+
"coverage": 0.8835078534031413,
|
| 76 |
+
"errorRate": 0.10222222222222223
|
| 77 |
+
},
|
| 78 |
+
{
|
| 79 |
+
"threshold": 0.8,
|
| 80 |
+
"accepted": 623,
|
| 81 |
+
"coverage": 0.8154450261780105,
|
| 82 |
+
"errorRate": 0.08186195826645265
|
| 83 |
+
},
|
| 84 |
+
{
|
| 85 |
+
"threshold": 0.9,
|
| 86 |
+
"accepted": 532,
|
| 87 |
+
"coverage": 0.6963350785340314,
|
| 88 |
+
"errorRate": 0.04887218045112782
|
| 89 |
+
},
|
| 90 |
+
{
|
| 91 |
+
"threshold": 0.95,
|
| 92 |
+
"accepted": 423,
|
| 93 |
+
"coverage": 0.5536649214659686,
|
| 94 |
+
"errorRate": 0.016548463356973995
|
| 95 |
+
},
|
| 96 |
+
{
|
| 97 |
+
"threshold": 0.99,
|
| 98 |
+
"accepted": 0,
|
| 99 |
+
"coverage": 0.0,
|
| 100 |
+
"errorRate": null
|
| 101 |
+
}
|
| 102 |
+
],
|
| 103 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here.",
|
| 104 |
+
"ordinalMeanAbsoluteError": 0.10620271886956209
|
| 105 |
+
},
|
| 106 |
+
"authoredRussian": {
|
| 107 |
+
"questions": 144,
|
| 108 |
+
"answered": 144,
|
| 109 |
+
"errors": 0,
|
| 110 |
+
"correct": 144,
|
| 111 |
+
"accuracyAllQuestions": 1.0,
|
| 112 |
+
"brierAnswered": 0.0009998268084272204,
|
| 113 |
+
"nllAnswered": 0.024466212500916604,
|
| 114 |
+
"ece10Answered": 0.024139216454452272,
|
| 115 |
+
"confidentWrongAt09": 0,
|
| 116 |
+
"coverageByFixedThreshold": [
|
| 117 |
+
{
|
| 118 |
+
"threshold": 0.5,
|
| 119 |
+
"accepted": 144,
|
| 120 |
+
"coverage": 1.0,
|
| 121 |
+
"errorRate": 0.0
|
| 122 |
+
},
|
| 123 |
+
{
|
| 124 |
+
"threshold": 0.7,
|
| 125 |
+
"accepted": 144,
|
| 126 |
+
"coverage": 1.0,
|
| 127 |
+
"errorRate": 0.0
|
| 128 |
+
},
|
| 129 |
+
{
|
| 130 |
+
"threshold": 0.8,
|
| 131 |
+
"accepted": 144,
|
| 132 |
+
"coverage": 1.0,
|
| 133 |
+
"errorRate": 0.0
|
| 134 |
+
},
|
| 135 |
+
{
|
| 136 |
+
"threshold": 0.9,
|
| 137 |
+
"accepted": 144,
|
| 138 |
+
"coverage": 1.0,
|
| 139 |
+
"errorRate": 0.0
|
| 140 |
+
},
|
| 141 |
+
{
|
| 142 |
+
"threshold": 0.95,
|
| 143 |
+
"accepted": 144,
|
| 144 |
+
"coverage": 1.0,
|
| 145 |
+
"errorRate": 0.0
|
| 146 |
+
},
|
| 147 |
+
{
|
| 148 |
+
"threshold": 0.99,
|
| 149 |
+
"accepted": 0,
|
| 150 |
+
"coverage": 0.0,
|
| 151 |
+
"errorRate": null
|
| 152 |
+
}
|
| 153 |
+
],
|
| 154 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here.",
|
| 155 |
+
"ordinalMeanAbsoluteError": 0.02282473410107361
|
| 156 |
+
},
|
| 157 |
+
"byType": {
|
| 158 |
+
"choice": {
|
| 159 |
+
"questions": 492,
|
| 160 |
+
"answered": 492,
|
| 161 |
+
"errors": 0,
|
| 162 |
+
"correct": 400,
|
| 163 |
+
"accuracyAllQuestions": 0.8130081300813008,
|
| 164 |
+
"brierAnswered": 0.2615263089500927,
|
| 165 |
+
"nllAnswered": 0.5247360934156637,
|
| 166 |
+
"ece10Answered": 0.06533461933177653,
|
| 167 |
+
"confidentWrongAt09": 20,
|
| 168 |
+
"coverageByFixedThreshold": [
|
| 169 |
+
{
|
| 170 |
+
"threshold": 0.5,
|
| 171 |
+
"accepted": 473,
|
| 172 |
+
"coverage": 0.9613821138211383,
|
| 173 |
+
"errorRate": 0.16279069767441862
|
| 174 |
+
},
|
| 175 |
+
{
|
| 176 |
+
"threshold": 0.7,
|
| 177 |
+
"accepted": 424,
|
| 178 |
+
"coverage": 0.8617886178861789,
|
| 179 |
+
"errorRate": 0.10377358490566038
|
| 180 |
+
},
|
| 181 |
+
{
|
| 182 |
+
"threshold": 0.8,
|
| 183 |
+
"accepted": 386,
|
| 184 |
+
"coverage": 0.7845528455284553,
|
| 185 |
+
"errorRate": 0.08031088082901554
|
| 186 |
+
},
|
| 187 |
+
{
|
| 188 |
+
"threshold": 0.9,
|
| 189 |
+
"accepted": 331,
|
| 190 |
+
"coverage": 0.6727642276422764,
|
| 191 |
+
"errorRate": 0.06042296072507553
|
| 192 |
+
},
|
| 193 |
+
{
|
| 194 |
+
"threshold": 0.95,
|
| 195 |
+
"accepted": 267,
|
| 196 |
+
"coverage": 0.5426829268292683,
|
| 197 |
+
"errorRate": 0.026217228464419477
|
| 198 |
+
},
|
| 199 |
+
{
|
| 200 |
+
"threshold": 0.99,
|
| 201 |
+
"accepted": 0,
|
| 202 |
+
"coverage": 0.0,
|
| 203 |
+
"errorRate": null
|
| 204 |
+
}
|
| 205 |
+
],
|
| 206 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 207 |
+
},
|
| 208 |
+
"noul": {
|
| 209 |
+
"questions": 328,
|
| 210 |
+
"answered": 328,
|
| 211 |
+
"errors": 0,
|
| 212 |
+
"correct": 298,
|
| 213 |
+
"accuracyAllQuestions": 0.9085365853658537,
|
| 214 |
+
"brierAnswered": 0.13871514597249474,
|
| 215 |
+
"nllAnswered": 0.22139737153892355,
|
| 216 |
+
"ece10Answered": 0.04994271263817423,
|
| 217 |
+
"confidentWrongAt09": 5,
|
| 218 |
+
"coverageByFixedThreshold": [
|
| 219 |
+
{
|
| 220 |
+
"threshold": 0.5,
|
| 221 |
+
"accepted": 328,
|
| 222 |
+
"coverage": 1.0,
|
| 223 |
+
"errorRate": 0.09146341463414634
|
| 224 |
+
},
|
| 225 |
+
{
|
| 226 |
+
"threshold": 0.7,
|
| 227 |
+
"accepted": 309,
|
| 228 |
+
"coverage": 0.9420731707317073,
|
| 229 |
+
"errorRate": 0.07766990291262135
|
| 230 |
+
},
|
| 231 |
+
{
|
| 232 |
+
"threshold": 0.8,
|
| 233 |
+
"accepted": 296,
|
| 234 |
+
"coverage": 0.9024390243902439,
|
| 235 |
+
"errorRate": 0.06418918918918919
|
| 236 |
+
},
|
| 237 |
+
{
|
| 238 |
+
"threshold": 0.9,
|
| 239 |
+
"accepted": 261,
|
| 240 |
+
"coverage": 0.7957317073170732,
|
| 241 |
+
"errorRate": 0.019157088122605363
|
| 242 |
+
},
|
| 243 |
+
{
|
| 244 |
+
"threshold": 0.95,
|
| 245 |
+
"accepted": 220,
|
| 246 |
+
"coverage": 0.6707317073170732,
|
| 247 |
+
"errorRate": 0.0
|
| 248 |
+
},
|
| 249 |
+
{
|
| 250 |
+
"threshold": 0.99,
|
| 251 |
+
"accepted": 0,
|
| 252 |
+
"coverage": 0.0,
|
| 253 |
+
"errorRate": null
|
| 254 |
+
}
|
| 255 |
+
],
|
| 256 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 257 |
+
},
|
| 258 |
+
"score": {
|
| 259 |
+
"questions": 88,
|
| 260 |
+
"answered": 88,
|
| 261 |
+
"errors": 0,
|
| 262 |
+
"correct": 85,
|
| 263 |
+
"accuracyAllQuestions": 0.9659090909090909,
|
| 264 |
+
"brierAnswered": 0.04486719339259727,
|
| 265 |
+
"nllAnswered": 0.10029860084375529,
|
| 266 |
+
"ece10Answered": 0.03799778408573792,
|
| 267 |
+
"confidentWrongAt09": 1,
|
| 268 |
+
"coverageByFixedThreshold": [
|
| 269 |
+
{
|
| 270 |
+
"threshold": 0.5,
|
| 271 |
+
"accepted": 88,
|
| 272 |
+
"coverage": 1.0,
|
| 273 |
+
"errorRate": 0.03409090909090909
|
| 274 |
+
},
|
| 275 |
+
{
|
| 276 |
+
"threshold": 0.7,
|
| 277 |
+
"accepted": 86,
|
| 278 |
+
"coverage": 0.9772727272727273,
|
| 279 |
+
"errorRate": 0.011627906976744186
|
| 280 |
+
},
|
| 281 |
+
{
|
| 282 |
+
"threshold": 0.8,
|
| 283 |
+
"accepted": 85,
|
| 284 |
+
"coverage": 0.9659090909090909,
|
| 285 |
+
"errorRate": 0.011764705882352941
|
| 286 |
+
},
|
| 287 |
+
{
|
| 288 |
+
"threshold": 0.9,
|
| 289 |
+
"accepted": 84,
|
| 290 |
+
"coverage": 0.9545454545454546,
|
| 291 |
+
"errorRate": 0.011904761904761904
|
| 292 |
+
},
|
| 293 |
+
{
|
| 294 |
+
"threshold": 0.95,
|
| 295 |
+
"accepted": 80,
|
| 296 |
+
"coverage": 0.9090909090909091,
|
| 297 |
+
"errorRate": 0.0
|
| 298 |
+
},
|
| 299 |
+
{
|
| 300 |
+
"threshold": 0.99,
|
| 301 |
+
"accepted": 0,
|
| 302 |
+
"coverage": 0.0,
|
| 303 |
+
"errorRate": null
|
| 304 |
+
}
|
| 305 |
+
],
|
| 306 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 307 |
+
}
|
| 308 |
+
},
|
| 309 |
+
"bySource": {
|
| 310 |
+
"authored-russian-typed": {
|
| 311 |
+
"questions": 144,
|
| 312 |
+
"answered": 144,
|
| 313 |
+
"errors": 0,
|
| 314 |
+
"correct": 144,
|
| 315 |
+
"accuracyAllQuestions": 1.0,
|
| 316 |
+
"brierAnswered": 0.0009998268084272204,
|
| 317 |
+
"nllAnswered": 0.024466212500916604,
|
| 318 |
+
"ece10Answered": 0.024139216454452272,
|
| 319 |
+
"confidentWrongAt09": 0,
|
| 320 |
+
"coverageByFixedThreshold": [
|
| 321 |
+
{
|
| 322 |
+
"threshold": 0.5,
|
| 323 |
+
"accepted": 144,
|
| 324 |
+
"coverage": 1.0,
|
| 325 |
+
"errorRate": 0.0
|
| 326 |
+
},
|
| 327 |
+
{
|
| 328 |
+
"threshold": 0.7,
|
| 329 |
+
"accepted": 144,
|
| 330 |
+
"coverage": 1.0,
|
| 331 |
+
"errorRate": 0.0
|
| 332 |
+
},
|
| 333 |
+
{
|
| 334 |
+
"threshold": 0.8,
|
| 335 |
+
"accepted": 144,
|
| 336 |
+
"coverage": 1.0,
|
| 337 |
+
"errorRate": 0.0
|
| 338 |
+
},
|
| 339 |
+
{
|
| 340 |
+
"threshold": 0.9,
|
| 341 |
+
"accepted": 144,
|
| 342 |
+
"coverage": 1.0,
|
| 343 |
+
"errorRate": 0.0
|
| 344 |
+
},
|
| 345 |
+
{
|
| 346 |
+
"threshold": 0.95,
|
| 347 |
+
"accepted": 144,
|
| 348 |
+
"coverage": 1.0,
|
| 349 |
+
"errorRate": 0.0
|
| 350 |
+
},
|
| 351 |
+
{
|
| 352 |
+
"threshold": 0.99,
|
| 353 |
+
"accepted": 0,
|
| 354 |
+
"coverage": 0.0,
|
| 355 |
+
"errorRate": null
|
| 356 |
+
}
|
| 357 |
+
],
|
| 358 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 359 |
+
},
|
| 360 |
+
"composition_holdout": {
|
| 361 |
+
"questions": 96,
|
| 362 |
+
"answered": 96,
|
| 363 |
+
"errors": 0,
|
| 364 |
+
"correct": 84,
|
| 365 |
+
"accuracyAllQuestions": 0.875,
|
| 366 |
+
"brierAnswered": 0.18821988619559113,
|
| 367 |
+
"nllAnswered": 0.31506340814867745,
|
| 368 |
+
"ece10Answered": 0.08715274769631735,
|
| 369 |
+
"confidentWrongAt09": 9,
|
| 370 |
+
"coverageByFixedThreshold": [
|
| 371 |
+
{
|
| 372 |
+
"threshold": 0.5,
|
| 373 |
+
"accepted": 96,
|
| 374 |
+
"coverage": 1.0,
|
| 375 |
+
"errorRate": 0.125
|
| 376 |
+
},
|
| 377 |
+
{
|
| 378 |
+
"threshold": 0.7,
|
| 379 |
+
"accepted": 92,
|
| 380 |
+
"coverage": 0.9583333333333334,
|
| 381 |
+
"errorRate": 0.09782608695652174
|
| 382 |
+
},
|
| 383 |
+
{
|
| 384 |
+
"threshold": 0.8,
|
| 385 |
+
"accepted": 92,
|
| 386 |
+
"coverage": 0.9583333333333334,
|
| 387 |
+
"errorRate": 0.09782608695652174
|
| 388 |
+
},
|
| 389 |
+
{
|
| 390 |
+
"threshold": 0.9,
|
| 391 |
+
"accepted": 92,
|
| 392 |
+
"coverage": 0.9583333333333334,
|
| 393 |
+
"errorRate": 0.09782608695652174
|
| 394 |
+
},
|
| 395 |
+
{
|
| 396 |
+
"threshold": 0.95,
|
| 397 |
+
"accepted": 86,
|
| 398 |
+
"coverage": 0.8958333333333334,
|
| 399 |
+
"errorRate": 0.03488372093023256
|
| 400 |
+
},
|
| 401 |
+
{
|
| 402 |
+
"threshold": 0.99,
|
| 403 |
+
"accepted": 0,
|
| 404 |
+
"coverage": 0.0,
|
| 405 |
+
"errorRate": null
|
| 406 |
+
}
|
| 407 |
+
],
|
| 408 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 409 |
+
},
|
| 410 |
+
"emotion": {
|
| 411 |
+
"questions": 116,
|
| 412 |
+
"answered": 116,
|
| 413 |
+
"errors": 0,
|
| 414 |
+
"correct": 69,
|
| 415 |
+
"accuracyAllQuestions": 0.5948275862068966,
|
| 416 |
+
"brierAnswered": 0.5245559470753833,
|
| 417 |
+
"nllAnswered": 1.0208149336604613,
|
| 418 |
+
"ece10Answered": 0.12016921458494689,
|
| 419 |
+
"confidentWrongAt09": 2,
|
| 420 |
+
"coverageByFixedThreshold": [
|
| 421 |
+
{
|
| 422 |
+
"threshold": 0.5,
|
| 423 |
+
"accepted": 101,
|
| 424 |
+
"coverage": 0.8706896551724138,
|
| 425 |
+
"errorRate": 0.3564356435643564
|
| 426 |
+
},
|
| 427 |
+
{
|
| 428 |
+
"threshold": 0.7,
|
| 429 |
+
"accepted": 70,
|
| 430 |
+
"coverage": 0.603448275862069,
|
| 431 |
+
"errorRate": 0.22857142857142856
|
| 432 |
+
},
|
| 433 |
+
{
|
| 434 |
+
"threshold": 0.8,
|
| 435 |
+
"accepted": 44,
|
| 436 |
+
"coverage": 0.3793103448275862,
|
| 437 |
+
"errorRate": 0.18181818181818182
|
| 438 |
+
},
|
| 439 |
+
{
|
| 440 |
+
"threshold": 0.9,
|
| 441 |
+
"accepted": 8,
|
| 442 |
+
"coverage": 0.06896551724137931,
|
| 443 |
+
"errorRate": 0.25
|
| 444 |
+
},
|
| 445 |
+
{
|
| 446 |
+
"threshold": 0.95,
|
| 447 |
+
"accepted": 0,
|
| 448 |
+
"coverage": 0.0,
|
| 449 |
+
"errorRate": null
|
| 450 |
+
},
|
| 451 |
+
{
|
| 452 |
+
"threshold": 0.99,
|
| 453 |
+
"accepted": 0,
|
| 454 |
+
"coverage": 0.0,
|
| 455 |
+
"errorRate": null
|
| 456 |
+
}
|
| 457 |
+
],
|
| 458 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 459 |
+
},
|
| 460 |
+
"legacy_holdout": {
|
| 461 |
+
"questions": 80,
|
| 462 |
+
"answered": 80,
|
| 463 |
+
"errors": 0,
|
| 464 |
+
"correct": 77,
|
| 465 |
+
"accuracyAllQuestions": 0.9625,
|
| 466 |
+
"brierAnswered": 0.04910351545555701,
|
| 467 |
+
"nllAnswered": 0.10416551178163372,
|
| 468 |
+
"ece10Answered": 0.03574970703743059,
|
| 469 |
+
"confidentWrongAt09": 1,
|
| 470 |
+
"coverageByFixedThreshold": [
|
| 471 |
+
{
|
| 472 |
+
"threshold": 0.5,
|
| 473 |
+
"accepted": 80,
|
| 474 |
+
"coverage": 1.0,
|
| 475 |
+
"errorRate": 0.0375
|
| 476 |
+
},
|
| 477 |
+
{
|
| 478 |
+
"threshold": 0.7,
|
| 479 |
+
"accepted": 78,
|
| 480 |
+
"coverage": 0.975,
|
| 481 |
+
"errorRate": 0.01282051282051282
|
| 482 |
+
},
|
| 483 |
+
{
|
| 484 |
+
"threshold": 0.8,
|
| 485 |
+
"accepted": 77,
|
| 486 |
+
"coverage": 0.9625,
|
| 487 |
+
"errorRate": 0.012987012987012988
|
| 488 |
+
},
|
| 489 |
+
{
|
| 490 |
+
"threshold": 0.9,
|
| 491 |
+
"accepted": 76,
|
| 492 |
+
"coverage": 0.95,
|
| 493 |
+
"errorRate": 0.013157894736842105
|
| 494 |
+
},
|
| 495 |
+
{
|
| 496 |
+
"threshold": 0.95,
|
| 497 |
+
"accepted": 72,
|
| 498 |
+
"coverage": 0.9,
|
| 499 |
+
"errorRate": 0.0
|
| 500 |
+
},
|
| 501 |
+
{
|
| 502 |
+
"threshold": 0.99,
|
| 503 |
+
"accepted": 0,
|
| 504 |
+
"coverage": 0.0,
|
| 505 |
+
"errorRate": null
|
| 506 |
+
}
|
| 507 |
+
],
|
| 508 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 509 |
+
},
|
| 510 |
+
"mmlu": {
|
| 511 |
+
"questions": 116,
|
| 512 |
+
"answered": 116,
|
| 513 |
+
"errors": 0,
|
| 514 |
+
"correct": 86,
|
| 515 |
+
"accuracyAllQuestions": 0.7413793103448276,
|
| 516 |
+
"brierAnswered": 0.39416899029285185,
|
| 517 |
+
"nllAnswered": 0.8394104030580065,
|
| 518 |
+
"ece10Answered": 0.12528513793721022,
|
| 519 |
+
"confidentWrongAt09": 9,
|
| 520 |
+
"coverageByFixedThreshold": [
|
| 521 |
+
{
|
| 522 |
+
"threshold": 0.5,
|
| 523 |
+
"accepted": 112,
|
| 524 |
+
"coverage": 0.9655172413793104,
|
| 525 |
+
"errorRate": 0.23214285714285715
|
| 526 |
+
},
|
| 527 |
+
{
|
| 528 |
+
"threshold": 0.7,
|
| 529 |
+
"accepted": 100,
|
| 530 |
+
"coverage": 0.8620689655172413,
|
| 531 |
+
"errorRate": 0.18
|
| 532 |
+
},
|
| 533 |
+
{
|
| 534 |
+
"threshold": 0.8,
|
| 535 |
+
"accepted": 91,
|
| 536 |
+
"coverage": 0.7844827586206896,
|
| 537 |
+
"errorRate": 0.14285714285714285
|
| 538 |
+
},
|
| 539 |
+
{
|
| 540 |
+
"threshold": 0.9,
|
| 541 |
+
"accepted": 77,
|
| 542 |
+
"coverage": 0.6637931034482759,
|
| 543 |
+
"errorRate": 0.11688311688311688
|
| 544 |
+
},
|
| 545 |
+
{
|
| 546 |
+
"threshold": 0.95,
|
| 547 |
+
"accepted": 42,
|
| 548 |
+
"coverage": 0.3620689655172414,
|
| 549 |
+
"errorRate": 0.09523809523809523
|
| 550 |
+
},
|
| 551 |
+
{
|
| 552 |
+
"threshold": 0.99,
|
| 553 |
+
"accepted": 0,
|
| 554 |
+
"coverage": 0.0,
|
| 555 |
+
"errorRate": null
|
| 556 |
+
}
|
| 557 |
+
],
|
| 558 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 559 |
+
},
|
| 560 |
+
"paws": {
|
| 561 |
+
"questions": 80,
|
| 562 |
+
"answered": 80,
|
| 563 |
+
"errors": 0,
|
| 564 |
+
"correct": 68,
|
| 565 |
+
"accuracyAllQuestions": 0.85,
|
| 566 |
+
"brierAnswered": 0.20695251414059662,
|
| 567 |
+
"nllAnswered": 0.3140944708626177,
|
| 568 |
+
"ece10Answered": 0.08749262200436604,
|
| 569 |
+
"confidentWrongAt09": 2,
|
| 570 |
+
"coverageByFixedThreshold": [
|
| 571 |
+
{
|
| 572 |
+
"threshold": 0.5,
|
| 573 |
+
"accepted": 80,
|
| 574 |
+
"coverage": 1.0,
|
| 575 |
+
"errorRate": 0.15
|
| 576 |
+
},
|
| 577 |
+
{
|
| 578 |
+
"threshold": 0.7,
|
| 579 |
+
"accepted": 75,
|
| 580 |
+
"coverage": 0.9375,
|
| 581 |
+
"errorRate": 0.12
|
| 582 |
+
},
|
| 583 |
+
{
|
| 584 |
+
"threshold": 0.8,
|
| 585 |
+
"accepted": 73,
|
| 586 |
+
"coverage": 0.9125,
|
| 587 |
+
"errorRate": 0.0958904109589041
|
| 588 |
+
},
|
| 589 |
+
{
|
| 590 |
+
"threshold": 0.9,
|
| 591 |
+
"accepted": 65,
|
| 592 |
+
"coverage": 0.8125,
|
| 593 |
+
"errorRate": 0.03076923076923077
|
| 594 |
+
},
|
| 595 |
+
{
|
| 596 |
+
"threshold": 0.95,
|
| 597 |
+
"accepted": 58,
|
| 598 |
+
"coverage": 0.725,
|
| 599 |
+
"errorRate": 0.0
|
| 600 |
+
},
|
| 601 |
+
{
|
| 602 |
+
"threshold": 0.99,
|
| 603 |
+
"accepted": 0,
|
| 604 |
+
"coverage": 0.0,
|
| 605 |
+
"errorRate": null
|
| 606 |
+
}
|
| 607 |
+
],
|
| 608 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 609 |
+
},
|
| 610 |
+
"qnli": {
|
| 611 |
+
"questions": 80,
|
| 612 |
+
"answered": 80,
|
| 613 |
+
"errors": 0,
|
| 614 |
+
"correct": 74,
|
| 615 |
+
"accuracyAllQuestions": 0.925,
|
| 616 |
+
"brierAnswered": 0.1185264487594759,
|
| 617 |
+
"nllAnswered": 0.19157792439378024,
|
| 618 |
+
"ece10Answered": 0.07130364771118254,
|
| 619 |
+
"confidentWrongAt09": 1,
|
| 620 |
+
"coverageByFixedThreshold": [
|
| 621 |
+
{
|
| 622 |
+
"threshold": 0.5,
|
| 623 |
+
"accepted": 80,
|
| 624 |
+
"coverage": 1.0,
|
| 625 |
+
"errorRate": 0.075
|
| 626 |
+
},
|
| 627 |
+
{
|
| 628 |
+
"threshold": 0.7,
|
| 629 |
+
"accepted": 79,
|
| 630 |
+
"coverage": 0.9875,
|
| 631 |
+
"errorRate": 0.0759493670886076
|
| 632 |
+
},
|
| 633 |
+
{
|
| 634 |
+
"threshold": 0.8,
|
| 635 |
+
"accepted": 77,
|
| 636 |
+
"coverage": 0.9625,
|
| 637 |
+
"errorRate": 0.06493506493506493
|
| 638 |
+
},
|
| 639 |
+
{
|
| 640 |
+
"threshold": 0.9,
|
| 641 |
+
"accepted": 71,
|
| 642 |
+
"coverage": 0.8875,
|
| 643 |
+
"errorRate": 0.014084507042253521
|
| 644 |
+
},
|
| 645 |
+
{
|
| 646 |
+
"threshold": 0.95,
|
| 647 |
+
"accepted": 60,
|
| 648 |
+
"coverage": 0.75,
|
| 649 |
+
"errorRate": 0.0
|
| 650 |
+
},
|
| 651 |
+
{
|
| 652 |
+
"threshold": 0.99,
|
| 653 |
+
"accepted": 0,
|
| 654 |
+
"coverage": 0.0,
|
| 655 |
+
"errorRate": null
|
| 656 |
+
}
|
| 657 |
+
],
|
| 658 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 659 |
+
},
|
| 660 |
+
"sciq": {
|
| 661 |
+
"questions": 116,
|
| 662 |
+
"answered": 116,
|
| 663 |
+
"errors": 0,
|
| 664 |
+
"correct": 113,
|
| 665 |
+
"accuracyAllQuestions": 0.9741379310344828,
|
| 666 |
+
"brierAnswered": 0.03409531064541125,
|
| 667 |
+
"nllAnswered": 0.09114362656929587,
|
| 668 |
+
"ece10Answered": 0.05454464085684041,
|
| 669 |
+
"confidentWrongAt09": 0,
|
| 670 |
+
"coverageByFixedThreshold": [
|
| 671 |
+
{
|
| 672 |
+
"threshold": 0.5,
|
| 673 |
+
"accepted": 116,
|
| 674 |
+
"coverage": 1.0,
|
| 675 |
+
"errorRate": 0.02586206896551724
|
| 676 |
+
},
|
| 677 |
+
{
|
| 678 |
+
"threshold": 0.7,
|
| 679 |
+
"accepted": 114,
|
| 680 |
+
"coverage": 0.9827586206896551,
|
| 681 |
+
"errorRate": 0.008771929824561403
|
| 682 |
+
},
|
| 683 |
+
{
|
| 684 |
+
"threshold": 0.8,
|
| 685 |
+
"accepted": 111,
|
| 686 |
+
"coverage": 0.9568965517241379,
|
| 687 |
+
"errorRate": 0.009009009009009009
|
| 688 |
+
},
|
| 689 |
+
{
|
| 690 |
+
"threshold": 0.9,
|
| 691 |
+
"accepted": 106,
|
| 692 |
+
"coverage": 0.9137931034482759,
|
| 693 |
+
"errorRate": 0.0
|
| 694 |
+
},
|
| 695 |
+
{
|
| 696 |
+
"threshold": 0.95,
|
| 697 |
+
"accepted": 91,
|
| 698 |
+
"coverage": 0.7844827586206896,
|
| 699 |
+
"errorRate": 0.0
|
| 700 |
+
},
|
| 701 |
+
{
|
| 702 |
+
"threshold": 0.99,
|
| 703 |
+
"accepted": 0,
|
| 704 |
+
"coverage": 0.0,
|
| 705 |
+
"errorRate": null
|
| 706 |
+
}
|
| 707 |
+
],
|
| 708 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 709 |
+
},
|
| 710 |
+
"tweet_offensive": {
|
| 711 |
+
"questions": 80,
|
| 712 |
+
"answered": 80,
|
| 713 |
+
"errors": 0,
|
| 714 |
+
"correct": 68,
|
| 715 |
+
"accuracyAllQuestions": 0.85,
|
| 716 |
+
"brierAnswered": 0.24263742159686036,
|
| 717 |
+
"nllAnswered": 0.38374648265869776,
|
| 718 |
+
"ece10Answered": 0.05576197181938556,
|
| 719 |
+
"confidentWrongAt09": 2,
|
| 720 |
+
"coverageByFixedThreshold": [
|
| 721 |
+
{
|
| 722 |
+
"threshold": 0.5,
|
| 723 |
+
"accepted": 80,
|
| 724 |
+
"coverage": 1.0,
|
| 725 |
+
"errorRate": 0.15
|
| 726 |
+
},
|
| 727 |
+
{
|
| 728 |
+
"threshold": 0.7,
|
| 729 |
+
"accepted": 67,
|
| 730 |
+
"coverage": 0.8375,
|
| 731 |
+
"errorRate": 0.13432835820895522
|
| 732 |
+
},
|
| 733 |
+
{
|
| 734 |
+
"threshold": 0.8,
|
| 735 |
+
"accepted": 58,
|
| 736 |
+
"coverage": 0.725,
|
| 737 |
+
"errorRate": 0.1206896551724138
|
| 738 |
+
},
|
| 739 |
+
{
|
| 740 |
+
"threshold": 0.9,
|
| 741 |
+
"accepted": 37,
|
| 742 |
+
"coverage": 0.4625,
|
| 743 |
+
"errorRate": 0.05405405405405406
|
| 744 |
+
},
|
| 745 |
+
{
|
| 746 |
+
"threshold": 0.95,
|
| 747 |
+
"accepted": 14,
|
| 748 |
+
"coverage": 0.175,
|
| 749 |
+
"errorRate": 0.0
|
| 750 |
+
},
|
| 751 |
+
{
|
| 752 |
+
"threshold": 0.99,
|
| 753 |
+
"accepted": 0,
|
| 754 |
+
"coverage": 0.0,
|
| 755 |
+
"errorRate": null
|
| 756 |
+
}
|
| 757 |
+
],
|
| 758 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 759 |
+
}
|
| 760 |
+
}
|
| 761 |
+
},
|
| 762 |
+
"Kev": {
|
| 763 |
+
"all": {
|
| 764 |
+
"questions": 908,
|
| 765 |
+
"answered": 908,
|
| 766 |
+
"errors": 0,
|
| 767 |
+
"correct": 756,
|
| 768 |
+
"accuracyAllQuestions": 0.8325991189427313,
|
| 769 |
+
"brierAnswered": 0.23049057438945586,
|
| 770 |
+
"nllAnswered": 0.42905102574468273,
|
| 771 |
+
"ece10Answered": 0.051600521203983195,
|
| 772 |
+
"confidentWrongAt09": 6,
|
| 773 |
+
"coverageByFixedThreshold": [
|
| 774 |
+
{
|
| 775 |
+
"threshold": 0.5,
|
| 776 |
+
"accepted": 828,
|
| 777 |
+
"coverage": 0.9118942731277533,
|
| 778 |
+
"errorRate": 0.12560386473429952
|
| 779 |
+
},
|
| 780 |
+
{
|
| 781 |
+
"threshold": 0.7,
|
| 782 |
+
"accepted": 667,
|
| 783 |
+
"coverage": 0.7345814977973568,
|
| 784 |
+
"errorRate": 0.05997001499250375
|
| 785 |
+
},
|
| 786 |
+
{
|
| 787 |
+
"threshold": 0.8,
|
| 788 |
+
"accepted": 531,
|
| 789 |
+
"coverage": 0.5848017621145375,
|
| 790 |
+
"errorRate": 0.04331450094161959
|
| 791 |
+
},
|
| 792 |
+
{
|
| 793 |
+
"threshold": 0.9,
|
| 794 |
+
"accepted": 368,
|
| 795 |
+
"coverage": 0.4052863436123348,
|
| 796 |
+
"errorRate": 0.016304347826086956
|
| 797 |
+
},
|
| 798 |
+
{
|
| 799 |
+
"threshold": 0.95,
|
| 800 |
+
"accepted": 276,
|
| 801 |
+
"coverage": 0.3039647577092511,
|
| 802 |
+
"errorRate": 0.007246376811594203
|
| 803 |
+
},
|
| 804 |
+
{
|
| 805 |
+
"threshold": 0.99,
|
| 806 |
+
"accepted": 88,
|
| 807 |
+
"coverage": 0.09691629955947137,
|
| 808 |
+
"errorRate": 0.0
|
| 809 |
+
}
|
| 810 |
+
],
|
| 811 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here.",
|
| 812 |
+
"ordinalMeanAbsoluteError": 0.3190549633819242
|
| 813 |
+
},
|
| 814 |
+
"publicTransfer": {
|
| 815 |
+
"questions": 764,
|
| 816 |
+
"answered": 764,
|
| 817 |
+
"errors": 0,
|
| 818 |
+
"correct": 613,
|
| 819 |
+
"accuracyAllQuestions": 0.8023560209424084,
|
| 820 |
+
"brierAnswered": 0.2543006283406479,
|
| 821 |
+
"nllAnswered": 0.460382624472629,
|
| 822 |
+
"ece10Answered": 0.033847807241890064,
|
| 823 |
+
"confidentWrongAt09": 6,
|
| 824 |
+
"coverageByFixedThreshold": [
|
| 825 |
+
{
|
| 826 |
+
"threshold": 0.5,
|
| 827 |
+
"accepted": 691,
|
| 828 |
+
"coverage": 0.9044502617801047,
|
| 829 |
+
"errorRate": 0.15050651230101303
|
| 830 |
+
},
|
| 831 |
+
{
|
| 832 |
+
"threshold": 0.7,
|
| 833 |
+
"accepted": 542,
|
| 834 |
+
"coverage": 0.7094240837696335,
|
| 835 |
+
"errorRate": 0.07380073800738007
|
| 836 |
+
},
|
| 837 |
+
{
|
| 838 |
+
"threshold": 0.8,
|
| 839 |
+
"accepted": 468,
|
| 840 |
+
"coverage": 0.612565445026178,
|
| 841 |
+
"errorRate": 0.049145299145299144
|
| 842 |
+
},
|
| 843 |
+
{
|
| 844 |
+
"threshold": 0.9,
|
| 845 |
+
"accepted": 350,
|
| 846 |
+
"coverage": 0.4581151832460733,
|
| 847 |
+
"errorRate": 0.017142857142857144
|
| 848 |
+
},
|
| 849 |
+
{
|
| 850 |
+
"threshold": 0.95,
|
| 851 |
+
"accepted": 262,
|
| 852 |
+
"coverage": 0.34293193717277487,
|
| 853 |
+
"errorRate": 0.007633587786259542
|
| 854 |
+
},
|
| 855 |
+
{
|
| 856 |
+
"threshold": 0.99,
|
| 857 |
+
"accepted": 87,
|
| 858 |
+
"coverage": 0.11387434554973822,
|
| 859 |
+
"errorRate": 0.0
|
| 860 |
+
}
|
| 861 |
+
],
|
| 862 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here.",
|
| 863 |
+
"ordinalMeanAbsoluteError": 0.34230038267246843
|
| 864 |
+
},
|
| 865 |
+
"authoredRussian": {
|
| 866 |
+
"questions": 144,
|
| 867 |
+
"answered": 144,
|
| 868 |
+
"errors": 0,
|
| 869 |
+
"correct": 143,
|
| 870 |
+
"accuracyAllQuestions": 0.9930555555555556,
|
| 871 |
+
"brierAnswered": 0.10416501037063129,
|
| 872 |
+
"nllAnswered": 0.2628194880491901,
|
| 873 |
+
"ece10Answered": 0.21249311017124553,
|
| 874 |
+
"confidentWrongAt09": 0,
|
| 875 |
+
"coverageByFixedThreshold": [
|
| 876 |
+
{
|
| 877 |
+
"threshold": 0.5,
|
| 878 |
+
"accepted": 137,
|
| 879 |
+
"coverage": 0.9513888888888888,
|
| 880 |
+
"errorRate": 0.0
|
| 881 |
+
},
|
| 882 |
+
{
|
| 883 |
+
"threshold": 0.7,
|
| 884 |
+
"accepted": 125,
|
| 885 |
+
"coverage": 0.8680555555555556,
|
| 886 |
+
"errorRate": 0.0
|
| 887 |
+
},
|
| 888 |
+
{
|
| 889 |
+
"threshold": 0.8,
|
| 890 |
+
"accepted": 63,
|
| 891 |
+
"coverage": 0.4375,
|
| 892 |
+
"errorRate": 0.0
|
| 893 |
+
},
|
| 894 |
+
{
|
| 895 |
+
"threshold": 0.9,
|
| 896 |
+
"accepted": 18,
|
| 897 |
+
"coverage": 0.125,
|
| 898 |
+
"errorRate": 0.0
|
| 899 |
+
},
|
| 900 |
+
{
|
| 901 |
+
"threshold": 0.95,
|
| 902 |
+
"accepted": 14,
|
| 903 |
+
"coverage": 0.09722222222222222,
|
| 904 |
+
"errorRate": 0.0
|
| 905 |
+
},
|
| 906 |
+
{
|
| 907 |
+
"threshold": 0.99,
|
| 908 |
+
"accepted": 1,
|
| 909 |
+
"coverage": 0.006944444444444444,
|
| 910 |
+
"errorRate": 0.0
|
| 911 |
+
}
|
| 912 |
+
],
|
| 913 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here.",
|
| 914 |
+
"ordinalMeanAbsoluteError": 0.29968378063980394
|
| 915 |
+
},
|
| 916 |
+
"byType": {
|
| 917 |
+
"choice": {
|
| 918 |
+
"questions": 492,
|
| 919 |
+
"answered": 492,
|
| 920 |
+
"errors": 0,
|
| 921 |
+
"correct": 400,
|
| 922 |
+
"accuracyAllQuestions": 0.8130081300813008,
|
| 923 |
+
"brierAnswered": 0.24971593029461644,
|
| 924 |
+
"nllAnswered": 0.5001996781390384,
|
| 925 |
+
"ece10Answered": 0.05950644744126973,
|
| 926 |
+
"confidentWrongAt09": 2,
|
| 927 |
+
"coverageByFixedThreshold": [
|
| 928 |
+
{
|
| 929 |
+
"threshold": 0.5,
|
| 930 |
+
"accepted": 421,
|
| 931 |
+
"coverage": 0.8556910569105691,
|
| 932 |
+
"errorRate": 0.11163895486935867
|
| 933 |
+
},
|
| 934 |
+
{
|
| 935 |
+
"threshold": 0.7,
|
| 936 |
+
"accepted": 345,
|
| 937 |
+
"coverage": 0.7012195121951219,
|
| 938 |
+
"errorRate": 0.04927536231884058
|
| 939 |
+
},
|
| 940 |
+
{
|
| 941 |
+
"threshold": 0.8,
|
| 942 |
+
"accepted": 276,
|
| 943 |
+
"coverage": 0.5609756097560976,
|
| 944 |
+
"errorRate": 0.036231884057971016
|
| 945 |
+
},
|
| 946 |
+
{
|
| 947 |
+
"threshold": 0.9,
|
| 948 |
+
"accepted": 193,
|
| 949 |
+
"coverage": 0.39227642276422764,
|
| 950 |
+
"errorRate": 0.010362694300518135
|
| 951 |
+
},
|
| 952 |
+
{
|
| 953 |
+
"threshold": 0.95,
|
| 954 |
+
"accepted": 144,
|
| 955 |
+
"coverage": 0.2926829268292683,
|
| 956 |
+
"errorRate": 0.0
|
| 957 |
+
},
|
| 958 |
+
{
|
| 959 |
+
"threshold": 0.99,
|
| 960 |
+
"accepted": 36,
|
| 961 |
+
"coverage": 0.07317073170731707,
|
| 962 |
+
"errorRate": 0.0
|
| 963 |
+
}
|
| 964 |
+
],
|
| 965 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 966 |
+
},
|
| 967 |
+
"noul": {
|
| 968 |
+
"questions": 328,
|
| 969 |
+
"answered": 328,
|
| 970 |
+
"errors": 0,
|
| 971 |
+
"correct": 284,
|
| 972 |
+
"accuracyAllQuestions": 0.8658536585365854,
|
| 973 |
+
"brierAnswered": 0.19400091681333753,
|
| 974 |
+
"nllAnswered": 0.3162898738046143,
|
| 975 |
+
"ece10Answered": 0.03553192036633847,
|
| 976 |
+
"confidentWrongAt09": 3,
|
| 977 |
+
"coverageByFixedThreshold": [
|
| 978 |
+
{
|
| 979 |
+
"threshold": 0.5,
|
| 980 |
+
"accepted": 328,
|
| 981 |
+
"coverage": 1.0,
|
| 982 |
+
"errorRate": 0.13414634146341464
|
| 983 |
+
},
|
| 984 |
+
{
|
| 985 |
+
"threshold": 0.7,
|
| 986 |
+
"accepted": 264,
|
| 987 |
+
"coverage": 0.8048780487804879,
|
| 988 |
+
"errorRate": 0.07196969696969698
|
| 989 |
+
},
|
| 990 |
+
{
|
| 991 |
+
"threshold": 0.8,
|
| 992 |
+
"accepted": 216,
|
| 993 |
+
"coverage": 0.6585365853658537,
|
| 994 |
+
"errorRate": 0.05092592592592592
|
| 995 |
+
},
|
| 996 |
+
{
|
| 997 |
+
"threshold": 0.9,
|
| 998 |
+
"accepted": 154,
|
| 999 |
+
"coverage": 0.4695121951219512,
|
| 1000 |
+
"errorRate": 0.01948051948051948
|
| 1001 |
+
},
|
| 1002 |
+
{
|
| 1003 |
+
"threshold": 0.95,
|
| 1004 |
+
"accepted": 119,
|
| 1005 |
+
"coverage": 0.3628048780487805,
|
| 1006 |
+
"errorRate": 0.01680672268907563
|
| 1007 |
+
},
|
| 1008 |
+
{
|
| 1009 |
+
"threshold": 0.99,
|
| 1010 |
+
"accepted": 50,
|
| 1011 |
+
"coverage": 0.1524390243902439,
|
| 1012 |
+
"errorRate": 0.0
|
| 1013 |
+
}
|
| 1014 |
+
],
|
| 1015 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 1016 |
+
},
|
| 1017 |
+
"score": {
|
| 1018 |
+
"questions": 88,
|
| 1019 |
+
"answered": 88,
|
| 1020 |
+
"errors": 0,
|
| 1021 |
+
"correct": 72,
|
| 1022 |
+
"accuracyAllQuestions": 0.8181818181818182,
|
| 1023 |
+
"brierAnswered": 0.2590102627943176,
|
| 1024 |
+
"nllAnswered": 0.451556944589222,
|
| 1025 |
+
"ece10Answered": 0.08293436635062447,
|
| 1026 |
+
"confidentWrongAt09": 1,
|
| 1027 |
+
"coverageByFixedThreshold": [
|
| 1028 |
+
{
|
| 1029 |
+
"threshold": 0.5,
|
| 1030 |
+
"accepted": 79,
|
| 1031 |
+
"coverage": 0.8977272727272727,
|
| 1032 |
+
"errorRate": 0.16455696202531644
|
| 1033 |
+
},
|
| 1034 |
+
{
|
| 1035 |
+
"threshold": 0.7,
|
| 1036 |
+
"accepted": 58,
|
| 1037 |
+
"coverage": 0.6590909090909091,
|
| 1038 |
+
"errorRate": 0.06896551724137931
|
| 1039 |
+
},
|
| 1040 |
+
{
|
| 1041 |
+
"threshold": 0.8,
|
| 1042 |
+
"accepted": 39,
|
| 1043 |
+
"coverage": 0.4431818181818182,
|
| 1044 |
+
"errorRate": 0.05128205128205128
|
| 1045 |
+
},
|
| 1046 |
+
{
|
| 1047 |
+
"threshold": 0.9,
|
| 1048 |
+
"accepted": 21,
|
| 1049 |
+
"coverage": 0.23863636363636365,
|
| 1050 |
+
"errorRate": 0.047619047619047616
|
| 1051 |
+
},
|
| 1052 |
+
{
|
| 1053 |
+
"threshold": 0.95,
|
| 1054 |
+
"accepted": 13,
|
| 1055 |
+
"coverage": 0.14772727272727273,
|
| 1056 |
+
"errorRate": 0.0
|
| 1057 |
+
},
|
| 1058 |
+
{
|
| 1059 |
+
"threshold": 0.99,
|
| 1060 |
+
"accepted": 2,
|
| 1061 |
+
"coverage": 0.022727272727272728,
|
| 1062 |
+
"errorRate": 0.0
|
| 1063 |
+
}
|
| 1064 |
+
],
|
| 1065 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 1066 |
+
}
|
| 1067 |
+
},
|
| 1068 |
+
"bySource": {
|
| 1069 |
+
"authored-russian-typed": {
|
| 1070 |
+
"questions": 144,
|
| 1071 |
+
"answered": 144,
|
| 1072 |
+
"errors": 0,
|
| 1073 |
+
"correct": 143,
|
| 1074 |
+
"accuracyAllQuestions": 0.9930555555555556,
|
| 1075 |
+
"brierAnswered": 0.10416501037063129,
|
| 1076 |
+
"nllAnswered": 0.2628194880491901,
|
| 1077 |
+
"ece10Answered": 0.21249311017124553,
|
| 1078 |
+
"confidentWrongAt09": 0,
|
| 1079 |
+
"coverageByFixedThreshold": [
|
| 1080 |
+
{
|
| 1081 |
+
"threshold": 0.5,
|
| 1082 |
+
"accepted": 137,
|
| 1083 |
+
"coverage": 0.9513888888888888,
|
| 1084 |
+
"errorRate": 0.0
|
| 1085 |
+
},
|
| 1086 |
+
{
|
| 1087 |
+
"threshold": 0.7,
|
| 1088 |
+
"accepted": 125,
|
| 1089 |
+
"coverage": 0.8680555555555556,
|
| 1090 |
+
"errorRate": 0.0
|
| 1091 |
+
},
|
| 1092 |
+
{
|
| 1093 |
+
"threshold": 0.8,
|
| 1094 |
+
"accepted": 63,
|
| 1095 |
+
"coverage": 0.4375,
|
| 1096 |
+
"errorRate": 0.0
|
| 1097 |
+
},
|
| 1098 |
+
{
|
| 1099 |
+
"threshold": 0.9,
|
| 1100 |
+
"accepted": 18,
|
| 1101 |
+
"coverage": 0.125,
|
| 1102 |
+
"errorRate": 0.0
|
| 1103 |
+
},
|
| 1104 |
+
{
|
| 1105 |
+
"threshold": 0.95,
|
| 1106 |
+
"accepted": 14,
|
| 1107 |
+
"coverage": 0.09722222222222222,
|
| 1108 |
+
"errorRate": 0.0
|
| 1109 |
+
},
|
| 1110 |
+
{
|
| 1111 |
+
"threshold": 0.99,
|
| 1112 |
+
"accepted": 1,
|
| 1113 |
+
"coverage": 0.006944444444444444,
|
| 1114 |
+
"errorRate": 0.0
|
| 1115 |
+
}
|
| 1116 |
+
],
|
| 1117 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 1118 |
+
},
|
| 1119 |
+
"composition_holdout": {
|
| 1120 |
+
"questions": 96,
|
| 1121 |
+
"answered": 96,
|
| 1122 |
+
"errors": 0,
|
| 1123 |
+
"correct": 94,
|
| 1124 |
+
"accuracyAllQuestions": 0.9791666666666666,
|
| 1125 |
+
"brierAnswered": 0.07265750882301934,
|
| 1126 |
+
"nllAnswered": 0.14970972837197358,
|
| 1127 |
+
"ece10Answered": 0.07865301631331481,
|
| 1128 |
+
"confidentWrongAt09": 1,
|
| 1129 |
+
"coverageByFixedThreshold": [
|
| 1130 |
+
{
|
| 1131 |
+
"threshold": 0.5,
|
| 1132 |
+
"accepted": 96,
|
| 1133 |
+
"coverage": 1.0,
|
| 1134 |
+
"errorRate": 0.020833333333333332
|
| 1135 |
+
},
|
| 1136 |
+
{
|
| 1137 |
+
"threshold": 0.7,
|
| 1138 |
+
"accepted": 85,
|
| 1139 |
+
"coverage": 0.8854166666666666,
|
| 1140 |
+
"errorRate": 0.011764705882352941
|
| 1141 |
+
},
|
| 1142 |
+
{
|
| 1143 |
+
"threshold": 0.8,
|
| 1144 |
+
"accepted": 80,
|
| 1145 |
+
"coverage": 0.8333333333333334,
|
| 1146 |
+
"errorRate": 0.0125
|
| 1147 |
+
},
|
| 1148 |
+
{
|
| 1149 |
+
"threshold": 0.9,
|
| 1150 |
+
"accepted": 64,
|
| 1151 |
+
"coverage": 0.6666666666666666,
|
| 1152 |
+
"errorRate": 0.015625
|
| 1153 |
+
},
|
| 1154 |
+
{
|
| 1155 |
+
"threshold": 0.95,
|
| 1156 |
+
"accepted": 52,
|
| 1157 |
+
"coverage": 0.5416666666666666,
|
| 1158 |
+
"errorRate": 0.0
|
| 1159 |
+
},
|
| 1160 |
+
{
|
| 1161 |
+
"threshold": 0.99,
|
| 1162 |
+
"accepted": 13,
|
| 1163 |
+
"coverage": 0.13541666666666666,
|
| 1164 |
+
"errorRate": 0.0
|
| 1165 |
+
}
|
| 1166 |
+
],
|
| 1167 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 1168 |
+
},
|
| 1169 |
+
"emotion": {
|
| 1170 |
+
"questions": 116,
|
| 1171 |
+
"answered": 116,
|
| 1172 |
+
"errors": 0,
|
| 1173 |
+
"correct": 65,
|
| 1174 |
+
"accuracyAllQuestions": 0.5603448275862069,
|
| 1175 |
+
"brierAnswered": 0.5283768574785999,
|
| 1176 |
+
"nllAnswered": 1.0464486013956598,
|
| 1177 |
+
"ece10Answered": 0.10797638647123994,
|
| 1178 |
+
"confidentWrongAt09": 0,
|
| 1179 |
+
"coverageByFixedThreshold": [
|
| 1180 |
+
{
|
| 1181 |
+
"threshold": 0.5,
|
| 1182 |
+
"accepted": 89,
|
| 1183 |
+
"coverage": 0.7672413793103449,
|
| 1184 |
+
"errorRate": 0.33707865168539325
|
| 1185 |
+
},
|
| 1186 |
+
{
|
| 1187 |
+
"threshold": 0.7,
|
| 1188 |
+
"accepted": 48,
|
| 1189 |
+
"coverage": 0.41379310344827586,
|
| 1190 |
+
"errorRate": 0.20833333333333334
|
| 1191 |
+
},
|
| 1192 |
+
{
|
| 1193 |
+
"threshold": 0.8,
|
| 1194 |
+
"accepted": 34,
|
| 1195 |
+
"coverage": 0.29310344827586204,
|
| 1196 |
+
"errorRate": 0.17647058823529413
|
| 1197 |
+
},
|
| 1198 |
+
{
|
| 1199 |
+
"threshold": 0.9,
|
| 1200 |
+
"accepted": 12,
|
| 1201 |
+
"coverage": 0.10344827586206896,
|
| 1202 |
+
"errorRate": 0.0
|
| 1203 |
+
},
|
| 1204 |
+
{
|
| 1205 |
+
"threshold": 0.95,
|
| 1206 |
+
"accepted": 3,
|
| 1207 |
+
"coverage": 0.02586206896551724,
|
| 1208 |
+
"errorRate": 0.0
|
| 1209 |
+
},
|
| 1210 |
+
{
|
| 1211 |
+
"threshold": 0.99,
|
| 1212 |
+
"accepted": 0,
|
| 1213 |
+
"coverage": 0.0,
|
| 1214 |
+
"errorRate": null
|
| 1215 |
+
}
|
| 1216 |
+
],
|
| 1217 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 1218 |
+
},
|
| 1219 |
+
"legacy_holdout": {
|
| 1220 |
+
"questions": 80,
|
| 1221 |
+
"answered": 80,
|
| 1222 |
+
"errors": 0,
|
| 1223 |
+
"correct": 65,
|
| 1224 |
+
"accuracyAllQuestions": 0.8125,
|
| 1225 |
+
"brierAnswered": 0.18524784583046267,
|
| 1226 |
+
"nllAnswered": 0.28116834855200057,
|
| 1227 |
+
"ece10Answered": 0.08460090352211151,
|
| 1228 |
+
"confidentWrongAt09": 1,
|
| 1229 |
+
"coverageByFixedThreshold": [
|
| 1230 |
+
{
|
| 1231 |
+
"threshold": 0.5,
|
| 1232 |
+
"accepted": 78,
|
| 1233 |
+
"coverage": 0.975,
|
| 1234 |
+
"errorRate": 0.16666666666666666
|
| 1235 |
+
},
|
| 1236 |
+
{
|
| 1237 |
+
"threshold": 0.7,
|
| 1238 |
+
"accepted": 66,
|
| 1239 |
+
"coverage": 0.825,
|
| 1240 |
+
"errorRate": 0.06060606060606061
|
| 1241 |
+
},
|
| 1242 |
+
{
|
| 1243 |
+
"threshold": 0.8,
|
| 1244 |
+
"accepted": 63,
|
| 1245 |
+
"coverage": 0.7875,
|
| 1246 |
+
"errorRate": 0.031746031746031744
|
| 1247 |
+
},
|
| 1248 |
+
{
|
| 1249 |
+
"threshold": 0.9,
|
| 1250 |
+
"accepted": 58,
|
| 1251 |
+
"coverage": 0.725,
|
| 1252 |
+
"errorRate": 0.017241379310344827
|
| 1253 |
+
},
|
| 1254 |
+
{
|
| 1255 |
+
"threshold": 0.95,
|
| 1256 |
+
"accepted": 53,
|
| 1257 |
+
"coverage": 0.6625,
|
| 1258 |
+
"errorRate": 0.0
|
| 1259 |
+
},
|
| 1260 |
+
{
|
| 1261 |
+
"threshold": 0.99,
|
| 1262 |
+
"accepted": 39,
|
| 1263 |
+
"coverage": 0.4875,
|
| 1264 |
+
"errorRate": 0.0
|
| 1265 |
+
}
|
| 1266 |
+
],
|
| 1267 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 1268 |
+
},
|
| 1269 |
+
"mmlu": {
|
| 1270 |
+
"questions": 116,
|
| 1271 |
+
"answered": 116,
|
| 1272 |
+
"errors": 0,
|
| 1273 |
+
"correct": 79,
|
| 1274 |
+
"accuracyAllQuestions": 0.6810344827586207,
|
| 1275 |
+
"brierAnswered": 0.3964835619347099,
|
| 1276 |
+
"nllAnswered": 0.740201823820986,
|
| 1277 |
+
"ece10Answered": 0.06660618296076969,
|
| 1278 |
+
"confidentWrongAt09": 1,
|
| 1279 |
+
"coverageByFixedThreshold": [
|
| 1280 |
+
{
|
| 1281 |
+
"threshold": 0.5,
|
| 1282 |
+
"accepted": 74,
|
| 1283 |
+
"coverage": 0.6379310344827587,
|
| 1284 |
+
"errorRate": 0.17567567567567569
|
| 1285 |
+
},
|
| 1286 |
+
{
|
| 1287 |
+
"threshold": 0.7,
|
| 1288 |
+
"accepted": 59,
|
| 1289 |
+
"coverage": 0.5086206896551724,
|
| 1290 |
+
"errorRate": 0.1016949152542373
|
| 1291 |
+
},
|
| 1292 |
+
{
|
| 1293 |
+
"threshold": 0.8,
|
| 1294 |
+
"accepted": 44,
|
| 1295 |
+
"coverage": 0.3793103448275862,
|
| 1296 |
+
"errorRate": 0.06818181818181818
|
| 1297 |
+
},
|
| 1298 |
+
{
|
| 1299 |
+
"threshold": 0.9,
|
| 1300 |
+
"accepted": 28,
|
| 1301 |
+
"coverage": 0.2413793103448276,
|
| 1302 |
+
"errorRate": 0.03571428571428571
|
| 1303 |
+
},
|
| 1304 |
+
{
|
| 1305 |
+
"threshold": 0.95,
|
| 1306 |
+
"accepted": 15,
|
| 1307 |
+
"coverage": 0.12931034482758622,
|
| 1308 |
+
"errorRate": 0.0
|
| 1309 |
+
},
|
| 1310 |
+
{
|
| 1311 |
+
"threshold": 0.99,
|
| 1312 |
+
"accepted": 1,
|
| 1313 |
+
"coverage": 0.008620689655172414,
|
| 1314 |
+
"errorRate": 0.0
|
| 1315 |
+
}
|
| 1316 |
+
],
|
| 1317 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 1318 |
+
},
|
| 1319 |
+
"paws": {
|
| 1320 |
+
"questions": 80,
|
| 1321 |
+
"answered": 80,
|
| 1322 |
+
"errors": 0,
|
| 1323 |
+
"correct": 64,
|
| 1324 |
+
"accuracyAllQuestions": 0.8,
|
| 1325 |
+
"brierAnswered": 0.30918811576472116,
|
| 1326 |
+
"nllAnswered": 0.48629027291665167,
|
| 1327 |
+
"ece10Answered": 0.0669704500818625,
|
| 1328 |
+
"confidentWrongAt09": 2,
|
| 1329 |
+
"coverageByFixedThreshold": [
|
| 1330 |
+
{
|
| 1331 |
+
"threshold": 0.5,
|
| 1332 |
+
"accepted": 80,
|
| 1333 |
+
"coverage": 1.0,
|
| 1334 |
+
"errorRate": 0.2
|
| 1335 |
+
},
|
| 1336 |
+
{
|
| 1337 |
+
"threshold": 0.7,
|
| 1338 |
+
"accepted": 65,
|
| 1339 |
+
"coverage": 0.8125,
|
| 1340 |
+
"errorRate": 0.16923076923076924
|
| 1341 |
+
},
|
| 1342 |
+
{
|
| 1343 |
+
"threshold": 0.8,
|
| 1344 |
+
"accepted": 55,
|
| 1345 |
+
"coverage": 0.6875,
|
| 1346 |
+
"errorRate": 0.12727272727272726
|
| 1347 |
+
},
|
| 1348 |
+
{
|
| 1349 |
+
"threshold": 0.9,
|
| 1350 |
+
"accepted": 37,
|
| 1351 |
+
"coverage": 0.4625,
|
| 1352 |
+
"errorRate": 0.05405405405405406
|
| 1353 |
+
},
|
| 1354 |
+
{
|
| 1355 |
+
"threshold": 0.95,
|
| 1356 |
+
"accepted": 23,
|
| 1357 |
+
"coverage": 0.2875,
|
| 1358 |
+
"errorRate": 0.08695652173913043
|
| 1359 |
+
},
|
| 1360 |
+
{
|
| 1361 |
+
"threshold": 0.99,
|
| 1362 |
+
"accepted": 2,
|
| 1363 |
+
"coverage": 0.025,
|
| 1364 |
+
"errorRate": 0.0
|
| 1365 |
+
}
|
| 1366 |
+
],
|
| 1367 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 1368 |
+
},
|
| 1369 |
+
"qnli": {
|
| 1370 |
+
"questions": 80,
|
| 1371 |
+
"answered": 80,
|
| 1372 |
+
"errors": 0,
|
| 1373 |
+
"correct": 73,
|
| 1374 |
+
"accuracyAllQuestions": 0.9125,
|
| 1375 |
+
"brierAnswered": 0.14017388395851296,
|
| 1376 |
+
"nllAnswered": 0.2368256396206597,
|
| 1377 |
+
"ece10Answered": 0.028304750133247607,
|
| 1378 |
+
"confidentWrongAt09": 1,
|
| 1379 |
+
"coverageByFixedThreshold": [
|
| 1380 |
+
{
|
| 1381 |
+
"threshold": 0.5,
|
| 1382 |
+
"accepted": 80,
|
| 1383 |
+
"coverage": 1.0,
|
| 1384 |
+
"errorRate": 0.0875
|
| 1385 |
+
},
|
| 1386 |
+
{
|
| 1387 |
+
"threshold": 0.7,
|
| 1388 |
+
"accepted": 70,
|
| 1389 |
+
"coverage": 0.875,
|
| 1390 |
+
"errorRate": 0.05714285714285714
|
| 1391 |
+
},
|
| 1392 |
+
{
|
| 1393 |
+
"threshold": 0.8,
|
| 1394 |
+
"accepted": 63,
|
| 1395 |
+
"coverage": 0.7875,
|
| 1396 |
+
"errorRate": 0.031746031746031744
|
| 1397 |
+
},
|
| 1398 |
+
{
|
| 1399 |
+
"threshold": 0.9,
|
| 1400 |
+
"accepted": 50,
|
| 1401 |
+
"coverage": 0.625,
|
| 1402 |
+
"errorRate": 0.02
|
| 1403 |
+
},
|
| 1404 |
+
{
|
| 1405 |
+
"threshold": 0.95,
|
| 1406 |
+
"accepted": 38,
|
| 1407 |
+
"coverage": 0.475,
|
| 1408 |
+
"errorRate": 0.0
|
| 1409 |
+
},
|
| 1410 |
+
{
|
| 1411 |
+
"threshold": 0.99,
|
| 1412 |
+
"accepted": 10,
|
| 1413 |
+
"coverage": 0.125,
|
| 1414 |
+
"errorRate": 0.0
|
| 1415 |
+
}
|
| 1416 |
+
],
|
| 1417 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 1418 |
+
},
|
| 1419 |
+
"sciq": {
|
| 1420 |
+
"questions": 116,
|
| 1421 |
+
"answered": 116,
|
| 1422 |
+
"errors": 0,
|
| 1423 |
+
"correct": 114,
|
| 1424 |
+
"accuracyAllQuestions": 0.9827586206896551,
|
| 1425 |
+
"brierAnswered": 0.04432013428162752,
|
| 1426 |
+
"nllAnswered": 0.11124410753498573,
|
| 1427 |
+
"ece10Answered": 0.07043185982245576,
|
| 1428 |
+
"confidentWrongAt09": 0,
|
| 1429 |
+
"coverageByFixedThreshold": [
|
| 1430 |
+
{
|
| 1431 |
+
"threshold": 0.5,
|
| 1432 |
+
"accepted": 114,
|
| 1433 |
+
"coverage": 0.9827586206896551,
|
| 1434 |
+
"errorRate": 0.017543859649122806
|
| 1435 |
+
},
|
| 1436 |
+
{
|
| 1437 |
+
"threshold": 0.7,
|
| 1438 |
+
"accepted": 106,
|
| 1439 |
+
"coverage": 0.9137931034482759,
|
| 1440 |
+
"errorRate": 0.0
|
| 1441 |
+
},
|
| 1442 |
+
{
|
| 1443 |
+
"threshold": 0.8,
|
| 1444 |
+
"accepted": 98,
|
| 1445 |
+
"coverage": 0.8448275862068966,
|
| 1446 |
+
"errorRate": 0.0
|
| 1447 |
+
},
|
| 1448 |
+
{
|
| 1449 |
+
"threshold": 0.9,
|
| 1450 |
+
"accepted": 89,
|
| 1451 |
+
"coverage": 0.7672413793103449,
|
| 1452 |
+
"errorRate": 0.0
|
| 1453 |
+
},
|
| 1454 |
+
{
|
| 1455 |
+
"threshold": 0.95,
|
| 1456 |
+
"accepted": 74,
|
| 1457 |
+
"coverage": 0.6379310344827587,
|
| 1458 |
+
"errorRate": 0.0
|
| 1459 |
+
},
|
| 1460 |
+
{
|
| 1461 |
+
"threshold": 0.99,
|
| 1462 |
+
"accepted": 22,
|
| 1463 |
+
"coverage": 0.1896551724137931,
|
| 1464 |
+
"errorRate": 0.0
|
| 1465 |
+
}
|
| 1466 |
+
],
|
| 1467 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 1468 |
+
},
|
| 1469 |
+
"tweet_offensive": {
|
| 1470 |
+
"questions": 80,
|
| 1471 |
+
"answered": 80,
|
| 1472 |
+
"errors": 0,
|
| 1473 |
+
"correct": 59,
|
| 1474 |
+
"accuracyAllQuestions": 0.7375,
|
| 1475 |
+
"brierAnswered": 0.3014603416542088,
|
| 1476 |
+
"nllAnswered": 0.4607710560880617,
|
| 1477 |
+
"ece10Answered": 0.0959448419045657,
|
| 1478 |
+
"confidentWrongAt09": 0,
|
| 1479 |
+
"coverageByFixedThreshold": [
|
| 1480 |
+
{
|
| 1481 |
+
"threshold": 0.5,
|
| 1482 |
+
"accepted": 80,
|
| 1483 |
+
"coverage": 1.0,
|
| 1484 |
+
"errorRate": 0.2625
|
| 1485 |
+
},
|
| 1486 |
+
{
|
| 1487 |
+
"threshold": 0.7,
|
| 1488 |
+
"accepted": 43,
|
| 1489 |
+
"coverage": 0.5375,
|
| 1490 |
+
"errorRate": 0.09302325581395349
|
| 1491 |
+
},
|
| 1492 |
+
{
|
| 1493 |
+
"threshold": 0.8,
|
| 1494 |
+
"accepted": 31,
|
| 1495 |
+
"coverage": 0.3875,
|
| 1496 |
+
"errorRate": 0.06451612903225806
|
| 1497 |
+
},
|
| 1498 |
+
{
|
| 1499 |
+
"threshold": 0.9,
|
| 1500 |
+
"accepted": 12,
|
| 1501 |
+
"coverage": 0.15,
|
| 1502 |
+
"errorRate": 0.0
|
| 1503 |
+
},
|
| 1504 |
+
{
|
| 1505 |
+
"threshold": 0.95,
|
| 1506 |
+
"accepted": 4,
|
| 1507 |
+
"coverage": 0.05,
|
| 1508 |
+
"errorRate": 0.0
|
| 1509 |
+
},
|
| 1510 |
+
{
|
| 1511 |
+
"threshold": 0.99,
|
| 1512 |
+
"accepted": 0,
|
| 1513 |
+
"coverage": 0.0,
|
| 1514 |
+
"errorRate": null
|
| 1515 |
+
}
|
| 1516 |
+
],
|
| 1517 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 1518 |
+
}
|
| 1519 |
+
}
|
| 1520 |
+
},
|
| 1521 |
+
"Jev": {
|
| 1522 |
+
"all": {
|
| 1523 |
+
"questions": 908,
|
| 1524 |
+
"answered": 908,
|
| 1525 |
+
"errors": 0,
|
| 1526 |
+
"correct": 790,
|
| 1527 |
+
"accuracyAllQuestions": 0.8700440528634361,
|
| 1528 |
+
"brierAnswered": 0.18933414586842034,
|
| 1529 |
+
"nllAnswered": 0.8838910343827637,
|
| 1530 |
+
"ece10Answered": 0.04661137809816233,
|
| 1531 |
+
"confidentWrongAt09": 36,
|
| 1532 |
+
"coverageByFixedThreshold": [
|
| 1533 |
+
{
|
| 1534 |
+
"threshold": 0.5,
|
| 1535 |
+
"accepted": 900,
|
| 1536 |
+
"coverage": 0.9911894273127754,
|
| 1537 |
+
"errorRate": 0.12444444444444444
|
| 1538 |
+
},
|
| 1539 |
+
{
|
| 1540 |
+
"threshold": 0.7,
|
| 1541 |
+
"accepted": 807,
|
| 1542 |
+
"coverage": 0.8887665198237885,
|
| 1543 |
+
"errorRate": 0.08178438661710037
|
| 1544 |
+
},
|
| 1545 |
+
{
|
| 1546 |
+
"threshold": 0.8,
|
| 1547 |
+
"accepted": 757,
|
| 1548 |
+
"coverage": 0.8337004405286343,
|
| 1549 |
+
"errorRate": 0.06340819022457067
|
| 1550 |
+
},
|
| 1551 |
+
{
|
| 1552 |
+
"threshold": 0.9,
|
| 1553 |
+
"accepted": 695,
|
| 1554 |
+
"coverage": 0.7654185022026432,
|
| 1555 |
+
"errorRate": 0.051798561151079135
|
| 1556 |
+
},
|
| 1557 |
+
{
|
| 1558 |
+
"threshold": 0.95,
|
| 1559 |
+
"accepted": 629,
|
| 1560 |
+
"coverage": 0.6927312775330396,
|
| 1561 |
+
"errorRate": 0.04292527821939587
|
| 1562 |
+
},
|
| 1563 |
+
{
|
| 1564 |
+
"threshold": 0.99,
|
| 1565 |
+
"accepted": 443,
|
| 1566 |
+
"coverage": 0.4878854625550661,
|
| 1567 |
+
"errorRate": 0.040632054176072234
|
| 1568 |
+
}
|
| 1569 |
+
],
|
| 1570 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here.",
|
| 1571 |
+
"ordinalMeanAbsoluteError": 0.0325
|
| 1572 |
+
},
|
| 1573 |
+
"publicTransfer": {
|
| 1574 |
+
"questions": 764,
|
| 1575 |
+
"answered": 764,
|
| 1576 |
+
"errors": 0,
|
| 1577 |
+
"correct": 646,
|
| 1578 |
+
"accuracyAllQuestions": 0.8455497382198953,
|
| 1579 |
+
"brierAnswered": 0.22499791158183988,
|
| 1580 |
+
"nllAnswered": 1.049527189121179,
|
| 1581 |
+
"ece10Answered": 0.056352266116663954,
|
| 1582 |
+
"confidentWrongAt09": 36,
|
| 1583 |
+
"coverageByFixedThreshold": [
|
| 1584 |
+
{
|
| 1585 |
+
"threshold": 0.5,
|
| 1586 |
+
"accepted": 756,
|
| 1587 |
+
"coverage": 0.9895287958115183,
|
| 1588 |
+
"errorRate": 0.14814814814814814
|
| 1589 |
+
},
|
| 1590 |
+
{
|
| 1591 |
+
"threshold": 0.7,
|
| 1592 |
+
"accepted": 663,
|
| 1593 |
+
"coverage": 0.8678010471204188,
|
| 1594 |
+
"errorRate": 0.09954751131221719
|
| 1595 |
+
},
|
| 1596 |
+
{
|
| 1597 |
+
"threshold": 0.8,
|
| 1598 |
+
"accepted": 613,
|
| 1599 |
+
"coverage": 0.8023560209424084,
|
| 1600 |
+
"errorRate": 0.07830342577487764
|
| 1601 |
+
},
|
| 1602 |
+
{
|
| 1603 |
+
"threshold": 0.9,
|
| 1604 |
+
"accepted": 551,
|
| 1605 |
+
"coverage": 0.7212041884816754,
|
| 1606 |
+
"errorRate": 0.06533575317604355
|
| 1607 |
+
},
|
| 1608 |
+
{
|
| 1609 |
+
"threshold": 0.95,
|
| 1610 |
+
"accepted": 485,
|
| 1611 |
+
"coverage": 0.6348167539267016,
|
| 1612 |
+
"errorRate": 0.05567010309278351
|
| 1613 |
+
},
|
| 1614 |
+
{
|
| 1615 |
+
"threshold": 0.99,
|
| 1616 |
+
"accepted": 305,
|
| 1617 |
+
"coverage": 0.3992146596858639,
|
| 1618 |
+
"errorRate": 0.05901639344262295
|
| 1619 |
+
}
|
| 1620 |
+
],
|
| 1621 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here.",
|
| 1622 |
+
"ordinalMeanAbsoluteError": 0.07075000000000001
|
| 1623 |
+
},
|
| 1624 |
+
"authoredRussian": {
|
| 1625 |
+
"questions": 144,
|
| 1626 |
+
"answered": 144,
|
| 1627 |
+
"errors": 0,
|
| 1628 |
+
"correct": 144,
|
| 1629 |
+
"accuracyAllQuestions": 1.0,
|
| 1630 |
+
"brierAnswered": 0.00011805555555555571,
|
| 1631 |
+
"nllAnswered": 0.005099213409504898,
|
| 1632 |
+
"ece10Answered": 0.005069444444444349,
|
| 1633 |
+
"confidentWrongAt09": 0,
|
| 1634 |
+
"coverageByFixedThreshold": [
|
| 1635 |
+
{
|
| 1636 |
+
"threshold": 0.5,
|
| 1637 |
+
"accepted": 144,
|
| 1638 |
+
"coverage": 1.0,
|
| 1639 |
+
"errorRate": 0.0
|
| 1640 |
+
},
|
| 1641 |
+
{
|
| 1642 |
+
"threshold": 0.7,
|
| 1643 |
+
"accepted": 144,
|
| 1644 |
+
"coverage": 1.0,
|
| 1645 |
+
"errorRate": 0.0
|
| 1646 |
+
},
|
| 1647 |
+
{
|
| 1648 |
+
"threshold": 0.8,
|
| 1649 |
+
"accepted": 144,
|
| 1650 |
+
"coverage": 1.0,
|
| 1651 |
+
"errorRate": 0.0
|
| 1652 |
+
},
|
| 1653 |
+
{
|
| 1654 |
+
"threshold": 0.9,
|
| 1655 |
+
"accepted": 144,
|
| 1656 |
+
"coverage": 1.0,
|
| 1657 |
+
"errorRate": 0.0
|
| 1658 |
+
},
|
| 1659 |
+
{
|
| 1660 |
+
"threshold": 0.95,
|
| 1661 |
+
"accepted": 144,
|
| 1662 |
+
"coverage": 1.0,
|
| 1663 |
+
"errorRate": 0.0
|
| 1664 |
+
},
|
| 1665 |
+
{
|
| 1666 |
+
"threshold": 0.99,
|
| 1667 |
+
"accepted": 138,
|
| 1668 |
+
"coverage": 0.9583333333333334,
|
| 1669 |
+
"errorRate": 0.0
|
| 1670 |
+
}
|
| 1671 |
+
],
|
| 1672 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here.",
|
| 1673 |
+
"ordinalMeanAbsoluteError": 0.0006250000000000006
|
| 1674 |
+
},
|
| 1675 |
+
"byType": {
|
| 1676 |
+
"choice": {
|
| 1677 |
+
"questions": 492,
|
| 1678 |
+
"answered": 492,
|
| 1679 |
+
"errors": 0,
|
| 1680 |
+
"correct": 414,
|
| 1681 |
+
"accuracyAllQuestions": 0.8414634146341463,
|
| 1682 |
+
"brierAnswered": 0.2359069196108245,
|
| 1683 |
+
"nllAnswered": 1.4490444419682396,
|
| 1684 |
+
"ece10Answered": 0.08126652705921,
|
| 1685 |
+
"confidentWrongAt09": 28,
|
| 1686 |
+
"coverageByFixedThreshold": [
|
| 1687 |
+
{
|
| 1688 |
+
"threshold": 0.5,
|
| 1689 |
+
"accepted": 484,
|
| 1690 |
+
"coverage": 0.983739837398374,
|
| 1691 |
+
"errorRate": 0.1487603305785124
|
| 1692 |
+
},
|
| 1693 |
+
{
|
| 1694 |
+
"threshold": 0.7,
|
| 1695 |
+
"accepted": 437,
|
| 1696 |
+
"coverage": 0.8882113821138211,
|
| 1697 |
+
"errorRate": 0.10526315789473684
|
| 1698 |
+
},
|
| 1699 |
+
{
|
| 1700 |
+
"threshold": 0.8,
|
| 1701 |
+
"accepted": 411,
|
| 1702 |
+
"coverage": 0.8353658536585366,
|
| 1703 |
+
"errorRate": 0.0827250608272506
|
| 1704 |
+
},
|
| 1705 |
+
{
|
| 1706 |
+
"threshold": 0.9,
|
| 1707 |
+
"accepted": 384,
|
| 1708 |
+
"coverage": 0.7804878048780488,
|
| 1709 |
+
"errorRate": 0.07291666666666667
|
| 1710 |
+
},
|
| 1711 |
+
{
|
| 1712 |
+
"threshold": 0.95,
|
| 1713 |
+
"accepted": 362,
|
| 1714 |
+
"coverage": 0.7357723577235772,
|
| 1715 |
+
"errorRate": 0.06906077348066299
|
| 1716 |
+
},
|
| 1717 |
+
{
|
| 1718 |
+
"threshold": 0.99,
|
| 1719 |
+
"accepted": 320,
|
| 1720 |
+
"coverage": 0.6504065040650406,
|
| 1721 |
+
"errorRate": 0.05625
|
| 1722 |
+
}
|
| 1723 |
+
],
|
| 1724 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 1725 |
+
},
|
| 1726 |
+
"noul": {
|
| 1727 |
+
"questions": 328,
|
| 1728 |
+
"answered": 328,
|
| 1729 |
+
"errors": 0,
|
| 1730 |
+
"correct": 289,
|
| 1731 |
+
"accuracyAllQuestions": 0.8810975609756098,
|
| 1732 |
+
"brierAnswered": 0.16377439024390245,
|
| 1733 |
+
"nllAnswered": 0.26158678411952796,
|
| 1734 |
+
"ece10Answered": 0.015304878048780484,
|
| 1735 |
+
"confidentWrongAt09": 8,
|
| 1736 |
+
"coverageByFixedThreshold": [
|
| 1737 |
+
{
|
| 1738 |
+
"threshold": 0.5,
|
| 1739 |
+
"accepted": 328,
|
| 1740 |
+
"coverage": 1.0,
|
| 1741 |
+
"errorRate": 0.11890243902439024
|
| 1742 |
+
},
|
| 1743 |
+
{
|
| 1744 |
+
"threshold": 0.7,
|
| 1745 |
+
"accepted": 285,
|
| 1746 |
+
"coverage": 0.8689024390243902,
|
| 1747 |
+
"errorRate": 0.06666666666666667
|
| 1748 |
+
},
|
| 1749 |
+
{
|
| 1750 |
+
"threshold": 0.8,
|
| 1751 |
+
"accepted": 262,
|
| 1752 |
+
"coverage": 0.7987804878048781,
|
| 1753 |
+
"errorRate": 0.05343511450381679
|
| 1754 |
+
},
|
| 1755 |
+
{
|
| 1756 |
+
"threshold": 0.9,
|
| 1757 |
+
"accepted": 229,
|
| 1758 |
+
"coverage": 0.698170731707317,
|
| 1759 |
+
"errorRate": 0.034934497816593885
|
| 1760 |
+
},
|
| 1761 |
+
{
|
| 1762 |
+
"threshold": 0.95,
|
| 1763 |
+
"accepted": 186,
|
| 1764 |
+
"coverage": 0.5670731707317073,
|
| 1765 |
+
"errorRate": 0.010752688172043012
|
| 1766 |
+
},
|
| 1767 |
+
{
|
| 1768 |
+
"threshold": 0.99,
|
| 1769 |
+
"accepted": 55,
|
| 1770 |
+
"coverage": 0.1676829268292683,
|
| 1771 |
+
"errorRate": 0.0
|
| 1772 |
+
}
|
| 1773 |
+
],
|
| 1774 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 1775 |
+
},
|
| 1776 |
+
"score": {
|
| 1777 |
+
"questions": 88,
|
| 1778 |
+
"answered": 88,
|
| 1779 |
+
"errors": 0,
|
| 1780 |
+
"correct": 87,
|
| 1781 |
+
"accuracyAllQuestions": 0.9886363636363636,
|
| 1782 |
+
"brierAnswered": 0.02421818181818182,
|
| 1783 |
+
"nllAnswered": 0.043667370226935225,
|
| 1784 |
+
"ece10Answered": 0.03250000000000011,
|
| 1785 |
+
"confidentWrongAt09": 0,
|
| 1786 |
+
"coverageByFixedThreshold": [
|
| 1787 |
+
{
|
| 1788 |
+
"threshold": 0.5,
|
| 1789 |
+
"accepted": 88,
|
| 1790 |
+
"coverage": 1.0,
|
| 1791 |
+
"errorRate": 0.011363636363636364
|
| 1792 |
+
},
|
| 1793 |
+
{
|
| 1794 |
+
"threshold": 0.7,
|
| 1795 |
+
"accepted": 85,
|
| 1796 |
+
"coverage": 0.9659090909090909,
|
| 1797 |
+
"errorRate": 0.011764705882352941
|
| 1798 |
+
},
|
| 1799 |
+
{
|
| 1800 |
+
"threshold": 0.8,
|
| 1801 |
+
"accepted": 84,
|
| 1802 |
+
"coverage": 0.9545454545454546,
|
| 1803 |
+
"errorRate": 0.0
|
| 1804 |
+
},
|
| 1805 |
+
{
|
| 1806 |
+
"threshold": 0.9,
|
| 1807 |
+
"accepted": 82,
|
| 1808 |
+
"coverage": 0.9318181818181818,
|
| 1809 |
+
"errorRate": 0.0
|
| 1810 |
+
},
|
| 1811 |
+
{
|
| 1812 |
+
"threshold": 0.95,
|
| 1813 |
+
"accepted": 81,
|
| 1814 |
+
"coverage": 0.9204545454545454,
|
| 1815 |
+
"errorRate": 0.0
|
| 1816 |
+
},
|
| 1817 |
+
{
|
| 1818 |
+
"threshold": 0.99,
|
| 1819 |
+
"accepted": 68,
|
| 1820 |
+
"coverage": 0.7727272727272727,
|
| 1821 |
+
"errorRate": 0.0
|
| 1822 |
+
}
|
| 1823 |
+
],
|
| 1824 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 1825 |
+
}
|
| 1826 |
+
},
|
| 1827 |
+
"bySource": {
|
| 1828 |
+
"authored-russian-typed": {
|
| 1829 |
+
"questions": 144,
|
| 1830 |
+
"answered": 144,
|
| 1831 |
+
"errors": 0,
|
| 1832 |
+
"correct": 144,
|
| 1833 |
+
"accuracyAllQuestions": 1.0,
|
| 1834 |
+
"brierAnswered": 0.00011805555555555571,
|
| 1835 |
+
"nllAnswered": 0.005099213409504898,
|
| 1836 |
+
"ece10Answered": 0.005069444444444349,
|
| 1837 |
+
"confidentWrongAt09": 0,
|
| 1838 |
+
"coverageByFixedThreshold": [
|
| 1839 |
+
{
|
| 1840 |
+
"threshold": 0.5,
|
| 1841 |
+
"accepted": 144,
|
| 1842 |
+
"coverage": 1.0,
|
| 1843 |
+
"errorRate": 0.0
|
| 1844 |
+
},
|
| 1845 |
+
{
|
| 1846 |
+
"threshold": 0.7,
|
| 1847 |
+
"accepted": 144,
|
| 1848 |
+
"coverage": 1.0,
|
| 1849 |
+
"errorRate": 0.0
|
| 1850 |
+
},
|
| 1851 |
+
{
|
| 1852 |
+
"threshold": 0.8,
|
| 1853 |
+
"accepted": 144,
|
| 1854 |
+
"coverage": 1.0,
|
| 1855 |
+
"errorRate": 0.0
|
| 1856 |
+
},
|
| 1857 |
+
{
|
| 1858 |
+
"threshold": 0.9,
|
| 1859 |
+
"accepted": 144,
|
| 1860 |
+
"coverage": 1.0,
|
| 1861 |
+
"errorRate": 0.0
|
| 1862 |
+
},
|
| 1863 |
+
{
|
| 1864 |
+
"threshold": 0.95,
|
| 1865 |
+
"accepted": 144,
|
| 1866 |
+
"coverage": 1.0,
|
| 1867 |
+
"errorRate": 0.0
|
| 1868 |
+
},
|
| 1869 |
+
{
|
| 1870 |
+
"threshold": 0.99,
|
| 1871 |
+
"accepted": 138,
|
| 1872 |
+
"coverage": 0.9583333333333334,
|
| 1873 |
+
"errorRate": 0.0
|
| 1874 |
+
}
|
| 1875 |
+
],
|
| 1876 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 1877 |
+
},
|
| 1878 |
+
"composition_holdout": {
|
| 1879 |
+
"questions": 96,
|
| 1880 |
+
"answered": 96,
|
| 1881 |
+
"errors": 0,
|
| 1882 |
+
"correct": 86,
|
| 1883 |
+
"accuracyAllQuestions": 0.8958333333333334,
|
| 1884 |
+
"brierAnswered": 0.11480833333333335,
|
| 1885 |
+
"nllAnswered": 0.16559070638097498,
|
| 1886 |
+
"ece10Answered": 0.041458333333333354,
|
| 1887 |
+
"confidentWrongAt09": 1,
|
| 1888 |
+
"coverageByFixedThreshold": [
|
| 1889 |
+
{
|
| 1890 |
+
"threshold": 0.5,
|
| 1891 |
+
"accepted": 96,
|
| 1892 |
+
"coverage": 1.0,
|
| 1893 |
+
"errorRate": 0.10416666666666667
|
| 1894 |
+
},
|
| 1895 |
+
{
|
| 1896 |
+
"threshold": 0.7,
|
| 1897 |
+
"accepted": 81,
|
| 1898 |
+
"coverage": 0.84375,
|
| 1899 |
+
"errorRate": 0.012345679012345678
|
| 1900 |
+
},
|
| 1901 |
+
{
|
| 1902 |
+
"threshold": 0.8,
|
| 1903 |
+
"accepted": 80,
|
| 1904 |
+
"coverage": 0.8333333333333334,
|
| 1905 |
+
"errorRate": 0.0125
|
| 1906 |
+
},
|
| 1907 |
+
{
|
| 1908 |
+
"threshold": 0.9,
|
| 1909 |
+
"accepted": 77,
|
| 1910 |
+
"coverage": 0.8020833333333334,
|
| 1911 |
+
"errorRate": 0.012987012987012988
|
| 1912 |
+
},
|
| 1913 |
+
{
|
| 1914 |
+
"threshold": 0.95,
|
| 1915 |
+
"accepted": 73,
|
| 1916 |
+
"coverage": 0.7604166666666666,
|
| 1917 |
+
"errorRate": 0.0
|
| 1918 |
+
},
|
| 1919 |
+
{
|
| 1920 |
+
"threshold": 0.99,
|
| 1921 |
+
"accepted": 65,
|
| 1922 |
+
"coverage": 0.6770833333333334,
|
| 1923 |
+
"errorRate": 0.0
|
| 1924 |
+
}
|
| 1925 |
+
],
|
| 1926 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 1927 |
+
},
|
| 1928 |
+
"emotion": {
|
| 1929 |
+
"questions": 116,
|
| 1930 |
+
"answered": 116,
|
| 1931 |
+
"errors": 0,
|
| 1932 |
+
"correct": 70,
|
| 1933 |
+
"accuracyAllQuestions": 0.603448275862069,
|
| 1934 |
+
"brierAnswered": 0.5788285340341767,
|
| 1935 |
+
"nllAnswered": 3.146503884340211,
|
| 1936 |
+
"ece10Answered": 0.23599616858237546,
|
| 1937 |
+
"confidentWrongAt09": 14,
|
| 1938 |
+
"coverageByFixedThreshold": [
|
| 1939 |
+
{
|
| 1940 |
+
"threshold": 0.5,
|
| 1941 |
+
"accepted": 110,
|
| 1942 |
+
"coverage": 0.9482758620689655,
|
| 1943 |
+
"errorRate": 0.37272727272727274
|
| 1944 |
+
},
|
| 1945 |
+
{
|
| 1946 |
+
"threshold": 0.7,
|
| 1947 |
+
"accepted": 91,
|
| 1948 |
+
"coverage": 0.7844827586206896,
|
| 1949 |
+
"errorRate": 0.31868131868131866
|
| 1950 |
+
},
|
| 1951 |
+
{
|
| 1952 |
+
"threshold": 0.8,
|
| 1953 |
+
"accepted": 75,
|
| 1954 |
+
"coverage": 0.646551724137931,
|
| 1955 |
+
"errorRate": 0.25333333333333335
|
| 1956 |
+
},
|
| 1957 |
+
{
|
| 1958 |
+
"threshold": 0.9,
|
| 1959 |
+
"accepted": 59,
|
| 1960 |
+
"coverage": 0.5086206896551724,
|
| 1961 |
+
"errorRate": 0.23728813559322035
|
| 1962 |
+
},
|
| 1963 |
+
{
|
| 1964 |
+
"threshold": 0.95,
|
| 1965 |
+
"accepted": 52,
|
| 1966 |
+
"coverage": 0.4482758620689655,
|
| 1967 |
+
"errorRate": 0.25
|
| 1968 |
+
},
|
| 1969 |
+
{
|
| 1970 |
+
"threshold": 0.99,
|
| 1971 |
+
"accepted": 39,
|
| 1972 |
+
"coverage": 0.33620689655172414,
|
| 1973 |
+
"errorRate": 0.1794871794871795
|
| 1974 |
+
}
|
| 1975 |
+
],
|
| 1976 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 1977 |
+
},
|
| 1978 |
+
"legacy_holdout": {
|
| 1979 |
+
"questions": 80,
|
| 1980 |
+
"answered": 80,
|
| 1981 |
+
"errors": 0,
|
| 1982 |
+
"correct": 79,
|
| 1983 |
+
"accuracyAllQuestions": 0.9875,
|
| 1984 |
+
"brierAnswered": 0.02695,
|
| 1985 |
+
"nllAnswered": 0.05636262223757969,
|
| 1986 |
+
"ece10Answered": 0.043999999999999984,
|
| 1987 |
+
"confidentWrongAt09": 0,
|
| 1988 |
+
"coverageByFixedThreshold": [
|
| 1989 |
+
{
|
| 1990 |
+
"threshold": 0.5,
|
| 1991 |
+
"accepted": 80,
|
| 1992 |
+
"coverage": 1.0,
|
| 1993 |
+
"errorRate": 0.0125
|
| 1994 |
+
},
|
| 1995 |
+
{
|
| 1996 |
+
"threshold": 0.7,
|
| 1997 |
+
"accepted": 77,
|
| 1998 |
+
"coverage": 0.9625,
|
| 1999 |
+
"errorRate": 0.012987012987012988
|
| 2000 |
+
},
|
| 2001 |
+
{
|
| 2002 |
+
"threshold": 0.8,
|
| 2003 |
+
"accepted": 76,
|
| 2004 |
+
"coverage": 0.95,
|
| 2005 |
+
"errorRate": 0.0
|
| 2006 |
+
},
|
| 2007 |
+
{
|
| 2008 |
+
"threshold": 0.9,
|
| 2009 |
+
"accepted": 74,
|
| 2010 |
+
"coverage": 0.925,
|
| 2011 |
+
"errorRate": 0.0
|
| 2012 |
+
},
|
| 2013 |
+
{
|
| 2014 |
+
"threshold": 0.95,
|
| 2015 |
+
"accepted": 73,
|
| 2016 |
+
"coverage": 0.9125,
|
| 2017 |
+
"errorRate": 0.0
|
| 2018 |
+
},
|
| 2019 |
+
{
|
| 2020 |
+
"threshold": 0.99,
|
| 2021 |
+
"accepted": 31,
|
| 2022 |
+
"coverage": 0.3875,
|
| 2023 |
+
"errorRate": 0.0
|
| 2024 |
+
}
|
| 2025 |
+
],
|
| 2026 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 2027 |
+
},
|
| 2028 |
+
"mmlu": {
|
| 2029 |
+
"questions": 116,
|
| 2030 |
+
"answered": 116,
|
| 2031 |
+
"errors": 0,
|
| 2032 |
+
"correct": 97,
|
| 2033 |
+
"accuracyAllQuestions": 0.8362068965517241,
|
| 2034 |
+
"brierAnswered": 0.28328529741863073,
|
| 2035 |
+
"nllAnswered": 2.795124001333949,
|
| 2036 |
+
"ece10Answered": 0.12087251828631135,
|
| 2037 |
+
"confidentWrongAt09": 13,
|
| 2038 |
+
"coverageByFixedThreshold": [
|
| 2039 |
+
{
|
| 2040 |
+
"threshold": 0.5,
|
| 2041 |
+
"accepted": 114,
|
| 2042 |
+
"coverage": 0.9827586206896551,
|
| 2043 |
+
"errorRate": 0.15789473684210525
|
| 2044 |
+
},
|
| 2045 |
+
{
|
| 2046 |
+
"threshold": 0.7,
|
| 2047 |
+
"accepted": 105,
|
| 2048 |
+
"coverage": 0.9051724137931034,
|
| 2049 |
+
"errorRate": 0.13333333333333333
|
| 2050 |
+
},
|
| 2051 |
+
{
|
| 2052 |
+
"threshold": 0.8,
|
| 2053 |
+
"accepted": 99,
|
| 2054 |
+
"coverage": 0.853448275862069,
|
| 2055 |
+
"errorRate": 0.13131313131313133
|
| 2056 |
+
},
|
| 2057 |
+
{
|
| 2058 |
+
"threshold": 0.9,
|
| 2059 |
+
"accepted": 93,
|
| 2060 |
+
"coverage": 0.8017241379310345,
|
| 2061 |
+
"errorRate": 0.13978494623655913
|
| 2062 |
+
},
|
| 2063 |
+
{
|
| 2064 |
+
"threshold": 0.95,
|
| 2065 |
+
"accepted": 82,
|
| 2066 |
+
"coverage": 0.7068965517241379,
|
| 2067 |
+
"errorRate": 0.14634146341463414
|
| 2068 |
+
},
|
| 2069 |
+
{
|
| 2070 |
+
"threshold": 0.99,
|
| 2071 |
+
"accepted": 67,
|
| 2072 |
+
"coverage": 0.5775862068965517,
|
| 2073 |
+
"errorRate": 0.16417910447761194
|
| 2074 |
+
}
|
| 2075 |
+
],
|
| 2076 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 2077 |
+
},
|
| 2078 |
+
"paws": {
|
| 2079 |
+
"questions": 80,
|
| 2080 |
+
"answered": 80,
|
| 2081 |
+
"errors": 0,
|
| 2082 |
+
"correct": 63,
|
| 2083 |
+
"accuracyAllQuestions": 0.7875,
|
| 2084 |
+
"brierAnswered": 0.3104425,
|
| 2085 |
+
"nllAnswered": 0.4745888407794543,
|
| 2086 |
+
"ece10Answered": 0.0953749999999999,
|
| 2087 |
+
"confidentWrongAt09": 6,
|
| 2088 |
+
"coverageByFixedThreshold": [
|
| 2089 |
+
{
|
| 2090 |
+
"threshold": 0.5,
|
| 2091 |
+
"accepted": 80,
|
| 2092 |
+
"coverage": 1.0,
|
| 2093 |
+
"errorRate": 0.2125
|
| 2094 |
+
},
|
| 2095 |
+
{
|
| 2096 |
+
"threshold": 0.7,
|
| 2097 |
+
"accepted": 70,
|
| 2098 |
+
"coverage": 0.875,
|
| 2099 |
+
"errorRate": 0.17142857142857143
|
| 2100 |
+
},
|
| 2101 |
+
{
|
| 2102 |
+
"threshold": 0.8,
|
| 2103 |
+
"accepted": 67,
|
| 2104 |
+
"coverage": 0.8375,
|
| 2105 |
+
"errorRate": 0.16417910447761194
|
| 2106 |
+
},
|
| 2107 |
+
{
|
| 2108 |
+
"threshold": 0.9,
|
| 2109 |
+
"accepted": 56,
|
| 2110 |
+
"coverage": 0.7,
|
| 2111 |
+
"errorRate": 0.10714285714285714
|
| 2112 |
+
},
|
| 2113 |
+
{
|
| 2114 |
+
"threshold": 0.95,
|
| 2115 |
+
"accepted": 39,
|
| 2116 |
+
"coverage": 0.4875,
|
| 2117 |
+
"errorRate": 0.05128205128205128
|
| 2118 |
+
},
|
| 2119 |
+
{
|
| 2120 |
+
"threshold": 0.99,
|
| 2121 |
+
"accepted": 0,
|
| 2122 |
+
"coverage": 0.0,
|
| 2123 |
+
"errorRate": null
|
| 2124 |
+
}
|
| 2125 |
+
],
|
| 2126 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 2127 |
+
},
|
| 2128 |
+
"qnli": {
|
| 2129 |
+
"questions": 80,
|
| 2130 |
+
"answered": 80,
|
| 2131 |
+
"errors": 0,
|
| 2132 |
+
"correct": 73,
|
| 2133 |
+
"accuracyAllQuestions": 0.9125,
|
| 2134 |
+
"brierAnswered": 0.094585,
|
| 2135 |
+
"nllAnswered": 0.1699941228795627,
|
| 2136 |
+
"ece10Answered": 0.08575000000000002,
|
| 2137 |
+
"confidentWrongAt09": 0,
|
| 2138 |
+
"coverageByFixedThreshold": [
|
| 2139 |
+
{
|
| 2140 |
+
"threshold": 0.5,
|
| 2141 |
+
"accepted": 80,
|
| 2142 |
+
"coverage": 1.0,
|
| 2143 |
+
"errorRate": 0.0875
|
| 2144 |
+
},
|
| 2145 |
+
{
|
| 2146 |
+
"threshold": 0.7,
|
| 2147 |
+
"accepted": 71,
|
| 2148 |
+
"coverage": 0.8875,
|
| 2149 |
+
"errorRate": 0.014084507042253521
|
| 2150 |
+
},
|
| 2151 |
+
{
|
| 2152 |
+
"threshold": 0.8,
|
| 2153 |
+
"accepted": 63,
|
| 2154 |
+
"coverage": 0.7875,
|
| 2155 |
+
"errorRate": 0.0
|
| 2156 |
+
},
|
| 2157 |
+
{
|
| 2158 |
+
"threshold": 0.9,
|
| 2159 |
+
"accepted": 55,
|
| 2160 |
+
"coverage": 0.6875,
|
| 2161 |
+
"errorRate": 0.0
|
| 2162 |
+
},
|
| 2163 |
+
{
|
| 2164 |
+
"threshold": 0.95,
|
| 2165 |
+
"accepted": 42,
|
| 2166 |
+
"coverage": 0.525,
|
| 2167 |
+
"errorRate": 0.0
|
| 2168 |
+
},
|
| 2169 |
+
{
|
| 2170 |
+
"threshold": 0.99,
|
| 2171 |
+
"accepted": 1,
|
| 2172 |
+
"coverage": 0.0125,
|
| 2173 |
+
"errorRate": 0.0
|
| 2174 |
+
}
|
| 2175 |
+
],
|
| 2176 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 2177 |
+
},
|
| 2178 |
+
"sciq": {
|
| 2179 |
+
"questions": 116,
|
| 2180 |
+
"answered": 116,
|
| 2181 |
+
"errors": 0,
|
| 2182 |
+
"correct": 113,
|
| 2183 |
+
"accuracyAllQuestions": 0.9741379310344828,
|
| 2184 |
+
"brierAnswered": 0.04341551724137931,
|
| 2185 |
+
"nllAnswered": 0.0658923924477742,
|
| 2186 |
+
"ece10Answered": 0.01767241379310347,
|
| 2187 |
+
"confidentWrongAt09": 0,
|
| 2188 |
+
"coverageByFixedThreshold": [
|
| 2189 |
+
{
|
| 2190 |
+
"threshold": 0.5,
|
| 2191 |
+
"accepted": 116,
|
| 2192 |
+
"coverage": 1.0,
|
| 2193 |
+
"errorRate": 0.02586206896551724
|
| 2194 |
+
},
|
| 2195 |
+
{
|
| 2196 |
+
"threshold": 0.7,
|
| 2197 |
+
"accepted": 112,
|
| 2198 |
+
"coverage": 0.9655172413793104,
|
| 2199 |
+
"errorRate": 0.017857142857142856
|
| 2200 |
+
},
|
| 2201 |
+
{
|
| 2202 |
+
"threshold": 0.8,
|
| 2203 |
+
"accepted": 109,
|
| 2204 |
+
"coverage": 0.9396551724137931,
|
| 2205 |
+
"errorRate": 0.009174311926605505
|
| 2206 |
+
},
|
| 2207 |
+
{
|
| 2208 |
+
"threshold": 0.9,
|
| 2209 |
+
"accepted": 107,
|
| 2210 |
+
"coverage": 0.9224137931034483,
|
| 2211 |
+
"errorRate": 0.0
|
| 2212 |
+
},
|
| 2213 |
+
{
|
| 2214 |
+
"threshold": 0.95,
|
| 2215 |
+
"accepted": 107,
|
| 2216 |
+
"coverage": 0.9224137931034483,
|
| 2217 |
+
"errorRate": 0.0
|
| 2218 |
+
},
|
| 2219 |
+
{
|
| 2220 |
+
"threshold": 0.99,
|
| 2221 |
+
"accepted": 101,
|
| 2222 |
+
"coverage": 0.8706896551724138,
|
| 2223 |
+
"errorRate": 0.0
|
| 2224 |
+
}
|
| 2225 |
+
],
|
| 2226 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 2227 |
+
},
|
| 2228 |
+
"tweet_offensive": {
|
| 2229 |
+
"questions": 80,
|
| 2230 |
+
"answered": 80,
|
| 2231 |
+
"errors": 0,
|
| 2232 |
+
"correct": 65,
|
| 2233 |
+
"accuracyAllQuestions": 0.8125,
|
| 2234 |
+
"brierAnswered": 0.265965,
|
| 2235 |
+
"nllAnswered": 0.4124258192766881,
|
| 2236 |
+
"ece10Answered": 0.02999999999999997,
|
| 2237 |
+
"confidentWrongAt09": 2,
|
| 2238 |
+
"coverageByFixedThreshold": [
|
| 2239 |
+
{
|
| 2240 |
+
"threshold": 0.5,
|
| 2241 |
+
"accepted": 80,
|
| 2242 |
+
"coverage": 1.0,
|
| 2243 |
+
"errorRate": 0.1875
|
| 2244 |
+
},
|
| 2245 |
+
{
|
| 2246 |
+
"threshold": 0.7,
|
| 2247 |
+
"accepted": 56,
|
| 2248 |
+
"coverage": 0.7,
|
| 2249 |
+
"errorRate": 0.10714285714285714
|
| 2250 |
+
},
|
| 2251 |
+
{
|
| 2252 |
+
"threshold": 0.8,
|
| 2253 |
+
"accepted": 44,
|
| 2254 |
+
"coverage": 0.55,
|
| 2255 |
+
"errorRate": 0.06818181818181818
|
| 2256 |
+
},
|
| 2257 |
+
{
|
| 2258 |
+
"threshold": 0.9,
|
| 2259 |
+
"accepted": 30,
|
| 2260 |
+
"coverage": 0.375,
|
| 2261 |
+
"errorRate": 0.06666666666666667
|
| 2262 |
+
},
|
| 2263 |
+
{
|
| 2264 |
+
"threshold": 0.95,
|
| 2265 |
+
"accepted": 17,
|
| 2266 |
+
"coverage": 0.2125,
|
| 2267 |
+
"errorRate": 0.0
|
| 2268 |
+
},
|
| 2269 |
+
{
|
| 2270 |
+
"threshold": 0.99,
|
| 2271 |
+
"accepted": 1,
|
| 2272 |
+
"coverage": 0.0125,
|
| 2273 |
+
"errorRate": 0.0
|
| 2274 |
+
}
|
| 2275 |
+
],
|
| 2276 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 2277 |
+
}
|
| 2278 |
+
}
|
| 2279 |
+
}
|
| 2280 |
+
},
|
| 2281 |
+
"pairedOutcomes": {
|
| 2282 |
+
"Fev vs Kev": {
|
| 2283 |
+
"aOnlyCorrect": 63,
|
| 2284 |
+
"bOnlyCorrect": 36,
|
| 2285 |
+
"bothCorrect": 720,
|
| 2286 |
+
"neitherCorrect": 89
|
| 2287 |
+
},
|
| 2288 |
+
"Fev vs Jev": {
|
| 2289 |
+
"aOnlyCorrect": 28,
|
| 2290 |
+
"bOnlyCorrect": 35,
|
| 2291 |
+
"bothCorrect": 755,
|
| 2292 |
+
"neitherCorrect": 90
|
| 2293 |
+
},
|
| 2294 |
+
"Kev vs Jev": {
|
| 2295 |
+
"aOnlyCorrect": 35,
|
| 2296 |
+
"bOnlyCorrect": 69,
|
| 2297 |
+
"bothCorrect": 721,
|
| 2298 |
+
"neitherCorrect": 83
|
| 2299 |
+
}
|
| 2300 |
+
},
|
| 2301 |
+
"interpretation": "Descriptive comparison on fixed public development and repetitive authored inputs; no significance or unseen customer accuracy claim",
|
| 2302 |
+
"datasetSha256": "9266d6ea1ce83c9755a60cf48c7469b0fa3264c229e96f4142d683bada8fef91",
|
| 2303 |
+
"provenance": {
|
| 2304 |
+
"Fev": {
|
| 2305 |
+
"revision": "3ef077cd7cc3c97d696c18150358ab4de3a6884e",
|
| 2306 |
+
"codeZipSha256": "b3b7808d7c2cbe2ce26d4b31ee0dde2b3e894c6776537931b61cd173ef526f8e",
|
| 2307 |
+
"execution": "reference BF16 backbone, FP32 shared adapter unmerged",
|
| 2308 |
+
"dataSha256": "9266d6ea1ce83c9755a60cf48c7469b0fa3264c229e96f4142d683bada8fef91",
|
| 2309 |
+
"gpu": "NVIDIA A10G",
|
| 2310 |
+
"torch": "2.14.0+cu130",
|
| 2311 |
+
"questionIsolation": false
|
| 2312 |
+
},
|
| 2313 |
+
"Kev": {
|
| 2314 |
+
"revision": "6cfce5c2fa4b4bd64026336ab649c5ca78857d52",
|
| 2315 |
+
"codeRevision": "2074cc44256200bb37540103fa4572901faa10f3",
|
| 2316 |
+
"dataSha256": "9266d6ea1ce83c9755a60cf48c7469b0fa3264c229e96f4142d683bada8fef91",
|
| 2317 |
+
"precision": "BF16 merged, official eager SDPA; no graphs/fused",
|
| 2318 |
+
"gpu": "NVIDIA A10G",
|
| 2319 |
+
"torch": "2.8.0+cu128",
|
| 2320 |
+
"temperature": 2.406050072164233
|
| 2321 |
+
},
|
| 2322 |
+
"Jev": {
|
| 2323 |
+
"model": "jev-1.14.0",
|
| 2324 |
+
"endpoint": "https://api.typesafe.ai/v1/systemone",
|
| 2325 |
+
"records": 812,
|
| 2326 |
+
"datasetSha256": "9266d6ea1ce83c9755a60cf48c7469b0fa3264c229e96f4142d683bada8fef91",
|
| 2327 |
+
"estimatedCapUsd": 0.15,
|
| 2328 |
+
"conservativeByteTokenUpperEstimateUsd": 0.018575508,
|
| 2329 |
+
"priceUsdPerMillionInputTokens": 0.042,
|
| 2330 |
+
"noAutomaticRetries": true,
|
| 2331 |
+
"publicAuthoredInputsOnly": true
|
| 2332 |
+
}
|
| 2333 |
+
}
|
| 2334 |
+
}
|
evaluation/benchmarks/runtime.json
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"date": "2026-10-24",
|
| 3 |
+
"hardware": "NVIDIA A10G 24GB",
|
| 4 |
+
"weightsChanged": false,
|
| 5 |
+
"referenceDefault": true,
|
| 6 |
+
"executionOptions": {
|
| 7 |
+
"compile": "layer",
|
| 8 |
+
"cuda_graphs": false,
|
| 9 |
+
"prune_last": false,
|
| 10 |
+
"fp8": false,
|
| 11 |
+
"prefetch": false,
|
| 12 |
+
"token_budget": 16384
|
| 13 |
+
},
|
| 14 |
+
"finalDecisionRegression": {
|
| 15 |
+
"complete": true,
|
| 16 |
+
"cases": 303,
|
| 17 |
+
"finalMatches": 303,
|
| 18 |
+
"candidateSetChanges": 33,
|
| 19 |
+
"sameJobControls": 4,
|
| 20 |
+
"allSameJobControlsMatch": true,
|
| 21 |
+
"firstFinalMismatch": null,
|
| 22 |
+
"error": null,
|
| 23 |
+
"timings": {
|
| 24 |
+
"reference": {
|
| 25 |
+
"n": 15,
|
| 26 |
+
"medianMs": 2451.9707610002115,
|
| 27 |
+
"p95Ms": 2539.2162020002615
|
| 28 |
+
},
|
| 29 |
+
"compiled": {
|
| 30 |
+
"n": 15,
|
| 31 |
+
"medianMs": 1690.3312350000306,
|
| 32 |
+
"p95Ms": 1733.433431999856
|
| 33 |
+
}
|
| 34 |
+
},
|
| 35 |
+
"medianReductionPercent": 31.062341285403082,
|
| 36 |
+
"p95Ratio": 0.6826647650697676,
|
| 37 |
+
"peakAllocatedBytes": {
|
| 38 |
+
"reference": 9997422592,
|
| 39 |
+
"compiled": 10086032384
|
| 40 |
+
},
|
| 41 |
+
"allocatedMemoryRatio": 1.0088632636246573,
|
| 42 |
+
"localAdoptionEligible": true
|
| 43 |
+
},
|
| 44 |
+
"packagedIntegration": {
|
| 45 |
+
"routingControls": 6,
|
| 46 |
+
"routingMatches": 6,
|
| 47 |
+
"typedRequests": 4,
|
| 48 |
+
"typedQuestions": 12,
|
| 49 |
+
"allTypedWinnersMatch": true,
|
| 50 |
+
"maxProbabilityDelta": 0.0016201982392612946,
|
| 51 |
+
"realAuthenticatedSdkPlainAndZstdPassed": true,
|
| 52 |
+
"compiledPeakAllocatedBytes": 10086030848
|
| 53 |
+
},
|
| 54 |
+
"limitations": [
|
| 55 |
+
"15 warm timing samples per mode on three selected workloads; excludes loading, compilation, HTTP, queuing and concurrent throughput",
|
| 56 |
+
"Final decisions match on the regression; internal candidates and probabilities can differ",
|
| 57 |
+
"Packaged integration is a smaller additional check, not a second full regression",
|
| 58 |
+
"No production CPU capacity or operating-cost measurement"
|
| 59 |
+
]
|
| 60 |
+
}
|
evaluation/benchmarks/russian-typed.jsonl
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"id": "ru-public-0-0", "group": "ru-family-0", "domain": "astronomy", "source": "authored-russian-typed", "state": {"кружок": "astronomy", "текущее_указание": "открыть", "архив": "закрыть", "количество": 0}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "a"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": false}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 0}}}
|
| 2 |
+
{"id": "ru-public-0-1", "group": "ru-family-1", "domain": "astronomy", "source": "authored-russian-typed", "state": {"кружок": "astronomy", "текущее_указание": "закрыть", "архив": "открыть", "количество": 1}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "b"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": false}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 0}}}
|
| 3 |
+
{"id": "ru-public-0-2", "group": "ru-family-2", "domain": "astronomy", "source": "authored-russian-typed", "state": {"кружок": "astronomy", "текущее_указание": "оставить", "архив": "закрыть", "количество": 2}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "c"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": false}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 1}}}
|
| 4 |
+
{"id": "ru-public-0-3", "group": "ru-family-3", "domain": "astronomy", "source": "authored-russian-typed", "state": {"кружок": "astronomy", "текущее_указание": "открыть", "архив": "закрыть", "количество": 3}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "a"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 1}}}
|
| 5 |
+
{"id": "ru-public-0-4", "group": "ru-family-4", "domain": "astronomy", "source": "authored-russian-typed", "state": {"кружок": "astronomy", "текущее_указание": "закрыть", "архив": "открыть", "количество": 4}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "b"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 1}}}
|
| 6 |
+
{"id": "ru-public-0-5", "group": "ru-family-5", "domain": "astronomy", "source": "authored-russian-typed", "state": {"кружок": "astronomy", "текущее_указание": "оставить", "архив": "закрыть", "количество": 5}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "c"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 2}}}
|
| 7 |
+
{"id": "ru-public-0-6", "group": "ru-family-6", "domain": "astronomy", "source": "authored-russian-typed", "state": {"кружок": "astronomy", "текущее_указание": "открыть", "архив": "закрыть", "количество": 6}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "a"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 2}}}
|
| 8 |
+
{"id": "ru-public-0-7", "group": "ru-family-7", "domain": "astronomy", "source": "authored-russian-typed", "state": {"кружок": "astronomy", "текущее_указание": "закрыть", "архив": "открыть", "количество": 0}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "b"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": false}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 0}}}
|
| 9 |
+
{"id": "ru-public-1-0", "group": "ru-family-0", "domain": "orchestra", "source": "authored-russian-typed", "state": {"кружок": "orchestra", "текущее_указание": "открыть", "архив": "закрыть", "количество": 1}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "a"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": false}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 0}}}
|
| 10 |
+
{"id": "ru-public-1-1", "group": "ru-family-1", "domain": "orchestra", "source": "authored-russian-typed", "state": {"кружок": "orchestra", "текущее_указание": "закрыть", "архив": "открыть", "количество": 2}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "b"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": false}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 1}}}
|
| 11 |
+
{"id": "ru-public-1-2", "group": "ru-family-2", "domain": "orchestra", "source": "authored-russian-typed", "state": {"кружок": "orchestra", "текущее_указание": "оставить", "архив": "закрыть", "количество": 3}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "c"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 1}}}
|
| 12 |
+
{"id": "ru-public-1-3", "group": "ru-family-3", "domain": "orchestra", "source": "authored-russian-typed", "state": {"кружок": "orchestra", "текущее_указание": "открыть", "архив": "закрыть", "количество": 4}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "a"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 1}}}
|
| 13 |
+
{"id": "ru-public-1-4", "group": "ru-family-4", "domain": "orchestra", "source": "authored-russian-typed", "state": {"кружок": "orchestra", "текущее_указание": "закрыть", "архив": "открыть", "количество": 5}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "b"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 2}}}
|
| 14 |
+
{"id": "ru-public-1-5", "group": "ru-family-5", "domain": "orchestra", "source": "authored-russian-typed", "state": {"кружок": "orchestra", "текущее_указание": "оставить", "архив": "закрыть", "количество": 6}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "c"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 2}}}
|
| 15 |
+
{"id": "ru-public-1-6", "group": "ru-family-6", "domain": "orchestra", "source": "authored-russian-typed", "state": {"кружок": "orchestra", "текущее_указание": "открыть", "архив": "закрыть", "количество": 0}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "a"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": false}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 0}}}
|
| 16 |
+
{"id": "ru-public-1-7", "group": "ru-family-7", "domain": "orchestra", "source": "authored-russian-typed", "state": {"кружок": "orchestra", "текущее_указание": "закрыть", "архив": "открыть", "количество": 1}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "b"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": false}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 0}}}
|
| 17 |
+
{"id": "ru-public-2-0", "group": "ru-family-0", "domain": "bakery", "source": "authored-russian-typed", "state": {"кружок": "bakery", "текущее_указание": "открыть", "архив": "закрыть", "количество": 2}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "a"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": false}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 1}}}
|
| 18 |
+
{"id": "ru-public-2-1", "group": "ru-family-1", "domain": "bakery", "source": "authored-russian-typed", "state": {"кружок": "bakery", "текущее_указание": "закрыть", "архив": "открыть", "количество": 3}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "b"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 1}}}
|
| 19 |
+
{"id": "ru-public-2-2", "group": "ru-family-2", "domain": "bakery", "source": "authored-russian-typed", "state": {"кружок": "bakery", "текущее_указание": "оставить", "архив": "закрыть", "количество": 4}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "c"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 1}}}
|
| 20 |
+
{"id": "ru-public-2-3", "group": "ru-family-3", "domain": "bakery", "source": "authored-russian-typed", "state": {"кружок": "bakery", "текущее_указание": "открыть", "архив": "закрыть", "количество": 5}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "a"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 2}}}
|
| 21 |
+
{"id": "ru-public-2-4", "group": "ru-family-4", "domain": "bakery", "source": "authored-russian-typed", "state": {"кружок": "bakery", "текущее_указание": "закрыть", "архив": "открыть", "количество": 6}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "b"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 2}}}
|
| 22 |
+
{"id": "ru-public-2-5", "group": "ru-family-5", "domain": "bakery", "source": "authored-russian-typed", "state": {"кружок": "bakery", "текущее_указание": "оставить", "архив": "закрыть", "количество": 0}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "c"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": false}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 0}}}
|
| 23 |
+
{"id": "ru-public-2-6", "group": "ru-family-6", "domain": "bakery", "source": "authored-russian-typed", "state": {"кружок": "bakery", "текущее_указание": "открыть", "архив": "закрыть", "количество": 1}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "a"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": false}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 0}}}
|
| 24 |
+
{"id": "ru-public-2-7", "group": "ru-family-7", "domain": "bakery", "source": "authored-russian-typed", "state": {"кружок": "bakery", "текущее_указание": "закрыть", "архив": "открыть", "количество": 2}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "b"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": false}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 1}}}
|
| 25 |
+
{"id": "ru-public-3-0", "group": "ru-family-0", "domain": "robotics", "source": "authored-russian-typed", "state": {"кружок": "robotics", "текущее_указание": "открыть", "архив": "закрыть", "количество": 3}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "a"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 1}}}
|
| 26 |
+
{"id": "ru-public-3-1", "group": "ru-family-1", "domain": "robotics", "source": "authored-russian-typed", "state": {"кружок": "robotics", "текущее_указание": "закрыть", "архив": "открыть", "количество": 4}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "b"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 1}}}
|
| 27 |
+
{"id": "ru-public-3-2", "group": "ru-family-2", "domain": "robotics", "source": "authored-russian-typed", "state": {"кружок": "robotics", "текущее_указание": "оставить", "архив": "закрыть", "количество": 5}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "c"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 2}}}
|
| 28 |
+
{"id": "ru-public-3-3", "group": "ru-family-3", "domain": "robotics", "source": "authored-russian-typed", "state": {"кружок": "robotics", "текущее_указание": "открыть", "архив": "закрыть", "количество": 6}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "a"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 2}}}
|
| 29 |
+
{"id": "ru-public-3-4", "group": "ru-family-4", "domain": "robotics", "source": "authored-russian-typed", "state": {"кружок": "robotics", "текущее_указание": "закрыть", "архив": "открыть", "количество": 0}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "b"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": false}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 0}}}
|
| 30 |
+
{"id": "ru-public-3-5", "group": "ru-family-5", "domain": "robotics", "source": "authored-russian-typed", "state": {"кружок": "robotics", "текущее_указание": "оставить", "архив": "закрыть", "количество": 1}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "c"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": false}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 0}}}
|
| 31 |
+
{"id": "ru-public-3-6", "group": "ru-family-6", "domain": "robotics", "source": "authored-russian-typed", "state": {"кружок": "robotics", "текущее_указание": "открыть", "архив": "закрыть", "количество": 2}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "a"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": false}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 1}}}
|
| 32 |
+
{"id": "ru-public-3-7", "group": "ru-family-7", "domain": "robotics", "source": "authored-russian-typed", "state": {"кружок": "robotics", "текущее_указание": "закрыть", "архив": "открыть", "количество": 3}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "b"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 1}}}
|
| 33 |
+
{"id": "ru-public-4-0", "group": "ru-family-0", "domain": "aquarium", "source": "authored-russian-typed", "state": {"кружок": "aquarium", "текущее_указание": "открыть", "архив": "закрыть", "количество": 4}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "a"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 1}}}
|
| 34 |
+
{"id": "ru-public-4-1", "group": "ru-family-1", "domain": "aquarium", "source": "authored-russian-typed", "state": {"кружок": "aquarium", "текущее_указание": "закрыть", "архив": "открыть", "количество": 5}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "b"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 2}}}
|
| 35 |
+
{"id": "ru-public-4-2", "group": "ru-family-2", "domain": "aquarium", "source": "authored-russian-typed", "state": {"кружок": "aquarium", "текущее_указание": "оставить", "архив": "закрыть", "количество": 6}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "c"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 2}}}
|
| 36 |
+
{"id": "ru-public-4-3", "group": "ru-family-3", "domain": "aquarium", "source": "authored-russian-typed", "state": {"кружок": "aquarium", "текущее_указание": "открыть", "архив": "закрыть", "количество": 0}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "a"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": false}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 0}}}
|
| 37 |
+
{"id": "ru-public-4-4", "group": "ru-family-4", "domain": "aquarium", "source": "authored-russian-typed", "state": {"кружок": "aquarium", "текущее_указание": "закрыть", "архив": "открыть", "количество": 1}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "b"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": false}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 0}}}
|
| 38 |
+
{"id": "ru-public-4-5", "group": "ru-family-5", "domain": "aquarium", "source": "authored-russian-typed", "state": {"кружок": "aquarium", "текущее_указание": "оставить", "архив": "закрыть", "количество": 2}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "c"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": false}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 1}}}
|
| 39 |
+
{"id": "ru-public-4-6", "group": "ru-family-6", "domain": "aquarium", "source": "authored-russian-typed", "state": {"кружок": "aquarium", "текущее_указание": "открыть", "архив": "закрыть", "количество": 3}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "a"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 1}}}
|
| 40 |
+
{"id": "ru-public-4-7", "group": "ru-family-7", "domain": "aquarium", "source": "authored-russian-typed", "state": {"кружок": "aquarium", "текущее_указание": "закрыть", "архив": "открыть", "количество": 4}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "b"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 1}}}
|
| 41 |
+
{"id": "ru-public-5-0", "group": "ru-family-0", "domain": "photography", "source": "authored-russian-typed", "state": {"кружок": "photography", "текущее_указание": "открыть", "архив": "закрыть", "количество": 5}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "a"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 2}}}
|
| 42 |
+
{"id": "ru-public-5-1", "group": "ru-family-1", "domain": "photography", "source": "authored-russian-typed", "state": {"кружок": "photography", "текущее_указание": "закрыть", "архив": "открыть", "количество": 6}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "b"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 2}}}
|
| 43 |
+
{"id": "ru-public-5-2", "group": "ru-family-2", "domain": "photography", "source": "authored-russian-typed", "state": {"кружок": "photography", "текущее_указание": "оставить", "архив": "закрыть", "количество": 0}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "c"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": false}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 0}}}
|
| 44 |
+
{"id": "ru-public-5-3", "group": "ru-family-3", "domain": "photography", "source": "authored-russian-typed", "state": {"кружок": "photography", "текущее_указание": "открыть", "архив": "закрыть", "количество": 1}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "a"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": false}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 0}}}
|
| 45 |
+
{"id": "ru-public-5-4", "group": "ru-family-4", "domain": "photography", "source": "authored-russian-typed", "state": {"кружок": "photography", "текущее_указание": "закрыть", "архив": "открыть", "количество": 2}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "b"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": false}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 1}}}
|
| 46 |
+
{"id": "ru-public-5-5", "group": "ru-family-5", "domain": "photography", "source": "authored-russian-typed", "state": {"кружок": "photography", "текущее_указание": "оставить", "архив": "закрыть", "количество": 3}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "c"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 1}}}
|
| 47 |
+
{"id": "ru-public-5-6", "group": "ru-family-6", "domain": "photography", "source": "authored-russian-typed", "state": {"кружок": "photography", "текущее_указание": "открыть", "архив": "закрыть", "количество": 4}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "a"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 1}}}
|
| 48 |
+
{"id": "ru-public-5-7", "group": "ru-family-7", "domain": "photography", "source": "authored-russian-typed", "state": {"кружок": "photography", "текущее_указание": "закрыть", "архив": "открыть", "количество": 5}, "questions": {"action": {"type": "choice", "instructions": "Исполнить только текущее_указание, игнорировать архив.", "criteria": {"a": "открыть", "b": "закрыть", "c": "оставить"}, "label": "b"}, "enough": {"type": "noul", "instructions": "количество не меньше 3?", "label": true}, "level": {"type": "score", "instructions": "Уровень: мало если количество <2; средне если от 2 до 4 включительно; много если >4.", "criteria": ["мало", "средне", "много"], "label": 2}}}
|
evaluation/benchmarks/training-validation.json
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"purpose": "Mechanics-only smoke validation, not improved released weights or generalization",
|
| 3 |
+
"adopted": false,
|
| 4 |
+
"training": {
|
| 5 |
+
"steps": 6,
|
| 6 |
+
"seed": 42,
|
| 7 |
+
"learningRate": 2e-05,
|
| 8 |
+
"trainRecords": 28,
|
| 9 |
+
"trainDomains": [
|
| 10 |
+
"garden",
|
| 11 |
+
"library",
|
| 12 |
+
"museum",
|
| 13 |
+
"workshop"
|
| 14 |
+
],
|
| 15 |
+
"changedAdapterTensors": 240,
|
| 16 |
+
"backboneAndReadoutFrozen": true,
|
| 17 |
+
"seconds": 53.129103966
|
| 18 |
+
},
|
| 19 |
+
"developmentBefore": {
|
| 20 |
+
"questions": 12,
|
| 21 |
+
"answered": 12,
|
| 22 |
+
"errors": 0,
|
| 23 |
+
"correct": 12,
|
| 24 |
+
"accuracyAllQuestions": 1.0,
|
| 25 |
+
"brierAnswered": 0.0010257946398990957,
|
| 26 |
+
"nllAnswered": 0.024988250780074998,
|
| 27 |
+
"ece10Answered": 0.024658157503599232,
|
| 28 |
+
"confidentWrongAt09": 0,
|
| 29 |
+
"coverageByFixedThreshold": [
|
| 30 |
+
{
|
| 31 |
+
"threshold": 0.5,
|
| 32 |
+
"accepted": 12,
|
| 33 |
+
"coverage": 1.0,
|
| 34 |
+
"errorRate": 0.0
|
| 35 |
+
},
|
| 36 |
+
{
|
| 37 |
+
"threshold": 0.7,
|
| 38 |
+
"accepted": 12,
|
| 39 |
+
"coverage": 1.0,
|
| 40 |
+
"errorRate": 0.0
|
| 41 |
+
},
|
| 42 |
+
{
|
| 43 |
+
"threshold": 0.8,
|
| 44 |
+
"accepted": 12,
|
| 45 |
+
"coverage": 1.0,
|
| 46 |
+
"errorRate": 0.0
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"threshold": 0.9,
|
| 50 |
+
"accepted": 12,
|
| 51 |
+
"coverage": 1.0,
|
| 52 |
+
"errorRate": 0.0
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"threshold": 0.95,
|
| 56 |
+
"accepted": 12,
|
| 57 |
+
"coverage": 1.0,
|
| 58 |
+
"errorRate": 0.0
|
| 59 |
+
},
|
| 60 |
+
{
|
| 61 |
+
"threshold": 0.99,
|
| 62 |
+
"accepted": 0,
|
| 63 |
+
"coverage": 0.0,
|
| 64 |
+
"errorRate": null
|
| 65 |
+
}
|
| 66 |
+
],
|
| 67 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 68 |
+
},
|
| 69 |
+
"developmentAfter": {
|
| 70 |
+
"questions": 12,
|
| 71 |
+
"answered": 12,
|
| 72 |
+
"errors": 0,
|
| 73 |
+
"correct": 12,
|
| 74 |
+
"accuracyAllQuestions": 1.0,
|
| 75 |
+
"brierAnswered": 3.058330986685694e-05,
|
| 76 |
+
"nllAnswered": 0.004121613352848992,
|
| 77 |
+
"ece10Answered": 0.0041117529524682395,
|
| 78 |
+
"confidentWrongAt09": 0,
|
| 79 |
+
"coverageByFixedThreshold": [
|
| 80 |
+
{
|
| 81 |
+
"threshold": 0.5,
|
| 82 |
+
"accepted": 12,
|
| 83 |
+
"coverage": 1.0,
|
| 84 |
+
"errorRate": 0.0
|
| 85 |
+
},
|
| 86 |
+
{
|
| 87 |
+
"threshold": 0.7,
|
| 88 |
+
"accepted": 12,
|
| 89 |
+
"coverage": 1.0,
|
| 90 |
+
"errorRate": 0.0
|
| 91 |
+
},
|
| 92 |
+
{
|
| 93 |
+
"threshold": 0.8,
|
| 94 |
+
"accepted": 12,
|
| 95 |
+
"coverage": 1.0,
|
| 96 |
+
"errorRate": 0.0
|
| 97 |
+
},
|
| 98 |
+
{
|
| 99 |
+
"threshold": 0.9,
|
| 100 |
+
"accepted": 12,
|
| 101 |
+
"coverage": 1.0,
|
| 102 |
+
"errorRate": 0.0
|
| 103 |
+
},
|
| 104 |
+
{
|
| 105 |
+
"threshold": 0.95,
|
| 106 |
+
"accepted": 12,
|
| 107 |
+
"coverage": 1.0,
|
| 108 |
+
"errorRate": 0.0
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"threshold": 0.99,
|
| 112 |
+
"accepted": 12,
|
| 113 |
+
"coverage": 1.0,
|
| 114 |
+
"errorRate": 0.0
|
| 115 |
+
}
|
| 116 |
+
],
|
| 117 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 118 |
+
},
|
| 119 |
+
"reload": {
|
| 120 |
+
"complete": true,
|
| 121 |
+
"adopted": false,
|
| 122 |
+
"priorStrictGateFailed": true,
|
| 123 |
+
"all240AdapterTensorsBitExact": true,
|
| 124 |
+
"freshRepeat": {
|
| 125 |
+
"maxProbabilityDelta": 0.0,
|
| 126 |
+
"sameAllTopOptions": true,
|
| 127 |
+
"questions": 12
|
| 128 |
+
},
|
| 129 |
+
"crossProcessReload": {
|
| 130 |
+
"maxProbabilityDelta": 0.0,
|
| 131 |
+
"sameAllTopOptions": true,
|
| 132 |
+
"questions": 12
|
| 133 |
+
},
|
| 134 |
+
"originalStrictTolerancePassed": true,
|
| 135 |
+
"calibrationAndTestPassed": true,
|
| 136 |
+
"seconds": 196.0350929440001
|
| 137 |
+
},
|
| 138 |
+
"initialStrictReloadFailure": {
|
| 139 |
+
"maxProbabilityDelta": 0.00048164095202640755,
|
| 140 |
+
"tolerance": 1e-05,
|
| 141 |
+
"failed": true,
|
| 142 |
+
"interpretation": "Observed after compiled/HTTP work in the original process. Cause was not isolated. Preserved rather than erased by the successful fresh-process control."
|
| 143 |
+
},
|
| 144 |
+
"calibration": {
|
| 145 |
+
"records": 4,
|
| 146 |
+
"factor": 0.1353352832366127,
|
| 147 |
+
"factorBounds": [
|
| 148 |
+
0.1353352832366127,
|
| 149 |
+
7.38905609893065
|
| 150 |
+
],
|
| 151 |
+
"beforeNll": 0.003907485844979199,
|
| 152 |
+
"afterNll": 0.0,
|
| 153 |
+
"interpretation": "In-sample calibration fit; independent test needed before adoption",
|
| 154 |
+
"lowerBoundSelected": true
|
| 155 |
+
},
|
| 156 |
+
"toyTest": {
|
| 157 |
+
"questions": 12,
|
| 158 |
+
"answered": 12,
|
| 159 |
+
"errors": 0,
|
| 160 |
+
"correct": 12,
|
| 161 |
+
"accuracyAllQuestions": 1.0,
|
| 162 |
+
"brierAnswered": 3.9934072114968966e-38,
|
| 163 |
+
"nllAnswered": 0.0,
|
| 164 |
+
"ece10Answered": 0.0,
|
| 165 |
+
"confidentWrongAt09": 0,
|
| 166 |
+
"coverageByFixedThreshold": [
|
| 167 |
+
{
|
| 168 |
+
"threshold": 0.5,
|
| 169 |
+
"accepted": 12,
|
| 170 |
+
"coverage": 1.0,
|
| 171 |
+
"errorRate": 0.0
|
| 172 |
+
},
|
| 173 |
+
{
|
| 174 |
+
"threshold": 0.7,
|
| 175 |
+
"accepted": 12,
|
| 176 |
+
"coverage": 1.0,
|
| 177 |
+
"errorRate": 0.0
|
| 178 |
+
},
|
| 179 |
+
{
|
| 180 |
+
"threshold": 0.8,
|
| 181 |
+
"accepted": 12,
|
| 182 |
+
"coverage": 1.0,
|
| 183 |
+
"errorRate": 0.0
|
| 184 |
+
},
|
| 185 |
+
{
|
| 186 |
+
"threshold": 0.9,
|
| 187 |
+
"accepted": 12,
|
| 188 |
+
"coverage": 1.0,
|
| 189 |
+
"errorRate": 0.0
|
| 190 |
+
},
|
| 191 |
+
{
|
| 192 |
+
"threshold": 0.95,
|
| 193 |
+
"accepted": 12,
|
| 194 |
+
"coverage": 1.0,
|
| 195 |
+
"errorRate": 0.0
|
| 196 |
+
},
|
| 197 |
+
{
|
| 198 |
+
"threshold": 0.99,
|
| 199 |
+
"accepted": 12,
|
| 200 |
+
"coverage": 1.0,
|
| 201 |
+
"errorRate": 0.0
|
| 202 |
+
}
|
| 203 |
+
],
|
| 204 |
+
"coverageInterpretation": "Observed on this dataset; not a future error guarantee. Thresholds not tuned here."
|
| 205 |
+
},
|
| 206 |
+
"inputHashes": {
|
| 207 |
+
"train.jsonl": "cef745ddee9bf2e1d8f3639660441505bfb5aacf2c5cf9de5261a17ac1ba0bb5",
|
| 208 |
+
"development.jsonl": "c7da14026dbfd87b5dac9040c7b6ed33d3007b620aab018dc9c25ca5ce6d9059",
|
| 209 |
+
"calibration.jsonl": "b951c82c4263d6b0ec058b704f1d3d616ae85f483dc3d6dfdad49bed61c7481c",
|
| 210 |
+
"test.jsonl": "00baffcf3d9114d585294dd0955724ad9aff04475d3e9105584c9f1bbfbfbc8a"
|
| 211 |
+
},
|
| 212 |
+
"limitations": [
|
| 213 |
+
"Authored toy templates recur across partitions; this is not an independent cross-domain generalization test",
|
| 214 |
+
"Development was already 12/12 correct before training",
|
| 215 |
+
"Calibration chose its lower search bound on a tiny all-correct sample; this is not evidence of deployment calibration",
|
| 216 |
+
"No trained overlay or fitted temperature is adopted or included in the release"
|
| 217 |
+
]
|
| 218 |
+
}
|
examples/training.jsonl
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"id": "toy-0-0", "group": "toy-0-0", "domain": "library", "state": {"area": "library", "request": "нужен перенос", "reserved": 0, "limit": 3, "code": "T0-0"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "change"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 0}}}
|
| 2 |
+
{"id": "toy-0-1", "group": "toy-0-1", "domain": "library", "state": {"area": "library", "request": "нужна отмена", "reserved": 1, "limit": 4, "code": "T0-1"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "cancel"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 0}}}
|
| 3 |
+
{"id": "toy-0-2", "group": "toy-0-2", "domain": "library", "state": {"area": "library", "request": "нужна справка", "reserved": 2, "limit": 5, "code": "T0-2"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "info"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 1}}}
|
| 4 |
+
{"id": "toy-0-3", "group": "toy-0-3", "domain": "library", "state": {"area": "library", "request": "нужен перенос", "reserved": 3, "limit": 3, "code": "T0-3"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "change"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": false}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 1}}}
|
| 5 |
+
{"id": "toy-0-4", "group": "toy-0-4", "domain": "library", "state": {"area": "library", "request": "нужна отмена", "reserved": 4, "limit": 4, "code": "T0-4"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "cancel"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": false}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 1}}}
|
| 6 |
+
{"id": "toy-0-5", "group": "toy-0-5", "domain": "library", "state": {"area": "library", "request": "нужна справка", "reserved": 5, "limit": 5, "code": "T0-5"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "info"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": false}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 2}}}
|
| 7 |
+
{"id": "toy-0-6", "group": "toy-0-6", "domain": "library", "state": {"area": "library", "request": "нужен перенос", "reserved": 6, "limit": 3, "code": "T0-6"}, "questions": {"action": {"type": "choice", "instructions": "Выбе��ите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "change"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": false}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 2}}}
|
| 8 |
+
{"id": "toy-0-7", "group": "toy-0-7", "domain": "library", "state": {"area": "library", "request": "нужна отмена", "reserved": 0, "limit": 4, "code": "T0-7"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "cancel"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 0}}}
|
| 9 |
+
{"id": "toy-0-8", "group": "toy-0-8", "domain": "library", "state": {"area": "library", "request": "нужна справка", "reserved": 1, "limit": 5, "code": "T0-8"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "info"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 0}}}
|
| 10 |
+
{"id": "toy-0-9", "group": "toy-0-9", "domain": "library", "state": {"area": "library", "request": "нужен перенос", "reserved": 2, "limit": 3, "code": "T0-9"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "change"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 1}}}
|
| 11 |
+
{"id": "toy-1-0", "group": "toy-1-0", "domain": "workshop", "state": {"area": "workshop", "request": "нужен перенос", "reserved": 0, "limit": 3, "code": "T1-0"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "change"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 0}}}
|
| 12 |
+
{"id": "toy-1-1", "group": "toy-1-1", "domain": "workshop", "state": {"area": "workshop", "request": "нужна отмена", "reserved": 1, "limit": 4, "code": "T1-1"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "cancel"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 0}}}
|
| 13 |
+
{"id": "toy-1-2", "group": "toy-1-2", "domain": "workshop", "state": {"area": "workshop", "request": "нужна справка", "reserved": 2, "limit": 5, "code": "T1-2"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "info"}, "room": {"type": "noul", "instructions": "reserved с��рого меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 1}}}
|
| 14 |
+
{"id": "toy-1-3", "group": "toy-1-3", "domain": "workshop", "state": {"area": "workshop", "request": "нужен перенос", "reserved": 3, "limit": 3, "code": "T1-3"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "change"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": false}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 1}}}
|
| 15 |
+
{"id": "toy-1-4", "group": "toy-1-4", "domain": "workshop", "state": {"area": "workshop", "request": "нужна отмена", "reserved": 4, "limit": 4, "code": "T1-4"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "cancel"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": false}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 1}}}
|
| 16 |
+
{"id": "toy-1-5", "group": "toy-1-5", "domain": "workshop", "state": {"area": "workshop", "request": "нужна справка", "reserved": 5, "limit": 5, "code": "T1-5"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "info"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": false}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 2}}}
|
| 17 |
+
{"id": "toy-1-6", "group": "toy-1-6", "domain": "workshop", "state": {"area": "workshop", "request": "нужен перенос", "reserved": 6, "limit": 3, "code": "T1-6"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "change"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": false}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 2}}}
|
| 18 |
+
{"id": "toy-1-7", "group": "toy-1-7", "domain": "workshop", "state": {"area": "workshop", "request": "нужна отмена", "reserved": 0, "limit": 4, "code": "T1-7"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "cancel"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 0}}}
|
| 19 |
+
{"id": "toy-1-8", "group": "toy-1-8", "domain": "workshop", "state": {"area": "workshop", "request": "нужна справка", "reserved": 1, "limit": 5, "code": "T1-8"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "info"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 0}}}
|
| 20 |
+
{"id": "toy-1-9", "group": "toy-1-9", "domain": "workshop", "state": {"area": "workshop", "request": "нужен перенос", "reserved": 2, "limit": 3, "code": "T1-9"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "change"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 1}}}
|
| 21 |
+
{"id": "toy-2-0", "group": "toy-2-0", "domain": "garden", "state": {"area": "garden", "request": "нужен перенос", "reserved": 0, "limit": 3, "code": "T2-0"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "change"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 0}}}
|
| 22 |
+
{"id": "toy-2-1", "group": "toy-2-1", "domain": "garden", "state": {"area": "garden", "request": "нужна отмена", "reserved": 1, "limit": 4, "code": "T2-1"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "cancel"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 0}}}
|
| 23 |
+
{"id": "toy-2-2", "group": "toy-2-2", "domain": "garden", "state": {"area": "garden", "request": "нужна справка", "reserved": 2, "limit": 5, "code": "T2-2"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "info"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 1}}}
|
| 24 |
+
{"id": "toy-2-3", "group": "toy-2-3", "domain": "garden", "state": {"area": "garden", "request": "нужен перенос", "reserved": 3, "limit": 3, "code": "T2-3"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "change"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": false}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 1}}}
|
| 25 |
+
{"id": "toy-2-4", "group": "toy-2-4", "domain": "garden", "state": {"area": "garden", "request": "нужна отмена", "reserved": 4, "limit": 4, "code": "T2-4"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "cancel"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": false}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 1}}}
|
| 26 |
+
{"id": "toy-2-5", "group": "toy-2-5", "domain": "garden", "state": {"area": "garden", "request": "нужна справка", "reserved": 5, "limit": 5, "code": "T2-5"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "info"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": false}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 2}}}
|
| 27 |
+
{"id": "toy-2-6", "group": "toy-2-6", "domain": "garden", "state": {"area": "garden", "request": "нужен перенос", "reserved": 6, "limit": 3, "code": "T2-6"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "change"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": false}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 2}}}
|
| 28 |
+
{"id": "toy-2-7", "group": "toy-2-7", "domain": "garden", "state": {"area": "garden", "request": "нужна отмена", "reserved": 0, "limit": 4, "code": "T2-7"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "cancel"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 0}}}
|
| 29 |
+
{"id": "toy-2-8", "group": "toy-2-8", "domain": "garden", "state": {"area": "garden", "request": "нужна справка", "reserved": 1, "limit": 5, "code": "T2-8"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "info"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 0}}}
|
| 30 |
+
{"id": "toy-2-9", "group": "toy-2-9", "domain": "garden", "state": {"area": "garden", "request": "нужен перенос", "reserved": 2, "limit": 3, "code": "T2-9"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "change"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 1}}}
|
| 31 |
+
{"id": "toy-3-0", "group": "toy-3-0", "domain": "museum", "state": {"area": "museum", "request": "нужен перенос", "reserved": 0, "limit": 3, "code": "T3-0"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "change"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 0}}}
|
| 32 |
+
{"id": "toy-3-1", "group": "toy-3-1", "domain": "museum", "state": {"area": "museum", "request": "нужна отмена", "reserved": 1, "limit": 4, "code": "T3-1"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "cancel"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 0}}}
|
| 33 |
+
{"id": "toy-3-2", "group": "toy-3-2", "domain": "museum", "state": {"area": "museum", "request": "нужна справка", "reserved": 2, "limit": 5, "code": "T3-2"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "info"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 1}}}
|
| 34 |
+
{"id": "toy-3-3", "group": "toy-3-3", "domain": "museum", "state": {"area": "museum", "request": "нужен перенос", "reserved": 3, "limit": 3, "code": "T3-3"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "change"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": false}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 1}}}
|
| 35 |
+
{"id": "toy-3-4", "group": "toy-3-4", "domain": "museum", "state": {"area": "museum", "request": "нужна отмена", "reserved": 4, "limit": 4, "code": "T3-4"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "cancel"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": false}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 1}}}
|
| 36 |
+
{"id": "toy-3-5", "group": "toy-3-5", "domain": "museum", "state": {"area": "museum", "request": "нужна справка", "reserved": 5, "limit": 5, "code": "T3-5"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "info"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": false}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 2}}}
|
| 37 |
+
{"id": "toy-3-6", "group": "toy-3-6", "domain": "museum", "state": {"area": "museum", "request": "нужен перенос", "reserved": 6, "limit": 3, "code": "T3-6"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "change"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": false}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 2}}}
|
| 38 |
+
{"id": "toy-3-7", "group": "toy-3-7", "domain": "museum", "state": {"area": "museum", "request": "нужна отмена", "reserved": 0, "limit": 4, "code": "T3-7"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "cancel"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 0}}}
|
| 39 |
+
{"id": "toy-3-8", "group": "toy-3-8", "domain": "museum", "state": {"area": "museum", "request": "нужна справка", "reserved": 1, "limit": 5, "code": "T3-8"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "info"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 0}}}
|
| 40 |
+
{"id": "toy-3-9", "group": "toy-3-9", "domain": "museum", "state": {"area": "museum", "request": "нужен перенос", "reserved": 2, "limit": 3, "code": "T3-9"}, "questions": {"action": {"type": "choice", "instructions": "Выберите действие только по request.", "criteria": {"change": "нужен перенос", "cancel": "нужна отмена", "info": "нужна справка"}, "label": "change"}, "room": {"type": "noul", "instructions": "reserved строго меньше limit?", "label": true}, "load": {"type": "score", "instructions": "Оцените reserved: 0 — меньше 2; 1 — от 2 до 4 включительно; 2 — больше 4.", "criteria": ["меньше 2", "от 2 до 4 включительно", "больше 4"], "label": 1}}}
|
fev4b/adaptation.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Explicit adapter overlays; the downloaded package is never overwritten."""
|
| 2 |
+
import hashlib
|
| 3 |
+
import json
|
| 4 |
+
import math
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
from .integrity import file_sha256
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def base_identity(root):
|
| 11 |
+
manifest = json.loads((Path(root) / 'manifest.json').read_text())
|
| 12 |
+
weights = {k: v for k, v in manifest['files'].items()
|
| 13 |
+
if k.startswith(('checkpoint/', 'adapter/'))}
|
| 14 |
+
return hashlib.sha256(json.dumps(weights, sort_keys=True).encode()).hexdigest()
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def verify_adaptation(root, directory):
|
| 18 |
+
directory = Path(directory).resolve()
|
| 19 |
+
meta = json.loads((directory / 'adaptation.json').read_text())
|
| 20 |
+
if meta.get('format') != 'fev-adaptation-v1' or meta.get('baseIdentity') != base_identity(root):
|
| 21 |
+
raise ValueError('Adaptation does not match the packaged base and original adapter')
|
| 22 |
+
if set(meta.get('files', {})) != {'adapter_config.json', 'adapter_model.safetensors'}:
|
| 23 |
+
raise ValueError('Unexpected adaptation payload')
|
| 24 |
+
for name, digest in meta['files'].items():
|
| 25 |
+
path = directory / name
|
| 26 |
+
if path.is_symlink() or file_sha256(path) != digest:
|
| 27 |
+
raise ValueError('Adaptation checksum mismatch')
|
| 28 |
+
if json.loads((directory / 'adapter_config.json').read_text()) != json.loads((Path(root) / 'adapter/adapter_config.json').read_text()):
|
| 29 |
+
raise ValueError('Adapter architecture differs from the released checkpoint')
|
| 30 |
+
temperature = meta.get('temperature')
|
| 31 |
+
if type(temperature) not in (int, float) or not math.isfinite(temperature) or temperature <= 0:
|
| 32 |
+
raise ValueError('Invalid adaptation temperature')
|
| 33 |
+
return meta
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def load_adaptation(wrapper, directory):
|
| 37 |
+
from peft import get_peft_model_state_dict, set_peft_model_state_dict
|
| 38 |
+
from safetensors.torch import load_file
|
| 39 |
+
meta = verify_adaptation(wrapper.root, directory)
|
| 40 |
+
state = load_file(str(Path(directory) / 'adapter_model.safetensors'))
|
| 41 |
+
text = wrapper.model.backbone.language_model
|
| 42 |
+
existing = get_peft_model_state_dict(text)
|
| 43 |
+
if set(state) != set(existing) or any(state[k].shape != existing[k].shape for k in state):
|
| 44 |
+
raise ValueError('Adaptation tensor keys or shapes differ')
|
| 45 |
+
if any(v.dtype != existing[k].dtype or not v.isfinite().all() for k, v in state.items()):
|
| 46 |
+
raise ValueError('Adaptation tensor dtype or finite-value check failed')
|
| 47 |
+
restored = set_peft_model_state_dict(text, state)
|
| 48 |
+
if restored.unexpected_keys or any('lora_' in k for k in restored.missing_keys):
|
| 49 |
+
raise ValueError('Adaptation was not restored completely')
|
| 50 |
+
wrapper.model.temperature = meta['temperature']
|
| 51 |
+
return meta
|
fev4b/benchmark.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Score local Fev or a System One endpoint on labelled JSONL. No automatic retries."""
|
| 2 |
+
import argparse
|
| 3 |
+
import hashlib
|
| 4 |
+
import json
|
| 5 |
+
import os
|
| 6 |
+
import sys
|
| 7 |
+
import time
|
| 8 |
+
import urllib.error
|
| 9 |
+
import urllib.request
|
| 10 |
+
from datetime import datetime, timezone
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
|
| 13 |
+
from .data import load_records, request_of
|
| 14 |
+
from .metrics import report_for, rows_for
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def evaluate(records, predict, out, *, provenance):
|
| 18 |
+
out = Path(out)
|
| 19 |
+
out.mkdir(parents=True, exist_ok=False)
|
| 20 |
+
rows, latencies, served = [], [], set()
|
| 21 |
+
with (out / 'predictions.jsonl').open('x') as stream:
|
| 22 |
+
for record in records:
|
| 23 |
+
began = time.perf_counter()
|
| 24 |
+
try:
|
| 25 |
+
response = predict(request_of(record))
|
| 26 |
+
scored = rows_for(record, response)
|
| 27 |
+
served.add(response.get('model', 'unspecified'))
|
| 28 |
+
saved = {'id': record['id'], 'response': response}
|
| 29 |
+
except (ValueError, RuntimeError, KeyError, TypeError, urllib.error.URLError, TimeoutError) as error:
|
| 30 |
+
# Do not serialize exception text: remote errors can contain sensitive headers.
|
| 31 |
+
saved = {'id': record['id'], 'error': type(error).__name__}
|
| 32 |
+
scored = rows_for(record, error=saved['error'])
|
| 33 |
+
latency = (time.perf_counter() - began)*1000
|
| 34 |
+
saved['wallMs'] = latency
|
| 35 |
+
latencies.append(latency)
|
| 36 |
+
rows.extend(scored)
|
| 37 |
+
stream.write(json.dumps(saved, ensure_ascii=False, allow_nan=False) + '\n')
|
| 38 |
+
stream.flush()
|
| 39 |
+
result = {**report_for(rows), 'provenance': provenance, 'servedModels': sorted(served),
|
| 40 |
+
'records': len(records), 'startedUtc': provenance.get('startedUtc'),
|
| 41 |
+
'wallLatencyMs': {'median': sorted(latencies)[len(latencies)//2],
|
| 42 |
+
'p95': sorted(latencies)[min(len(latencies)-1, int(.95*len(latencies)))]},
|
| 43 |
+
'latencyMeaning': 'Sequential wall time including client overhead/cold calls; not model-only time or throughput'}
|
| 44 |
+
(out / 'rows.json').write_text(json.dumps(rows, ensure_ascii=False, indent=2) + '\n')
|
| 45 |
+
(out / 'report.json').write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
|
| 46 |
+
return result
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def endpoint_predictor(endpoint, model, api_key=None):
|
| 50 |
+
if not endpoint.startswith(('http://127.0.0.1:', 'http://localhost:', 'https://')):
|
| 51 |
+
raise ValueError('Use localhost HTTP or an HTTPS endpoint')
|
| 52 |
+
def predict(request):
|
| 53 |
+
headers = {'Content-Type': 'application/json'}
|
| 54 |
+
if api_key:
|
| 55 |
+
headers['Authorization'] = 'Bearer ' + api_key
|
| 56 |
+
req = urllib.request.Request(endpoint.rstrip('/') + '/v1/systemone',
|
| 57 |
+
data=json.dumps({'model': model, **request}, ensure_ascii=False).encode(), headers=headers)
|
| 58 |
+
with urllib.request.urlopen(req, timeout=120) as response:
|
| 59 |
+
return json.load(response)
|
| 60 |
+
return predict
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def main():
|
| 64 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 65 |
+
parser.add_argument('--data', type=Path, required=True)
|
| 66 |
+
parser.add_argument('--out', type=Path, required=True)
|
| 67 |
+
source = parser.add_mutually_exclusive_group(required=True)
|
| 68 |
+
source.add_argument('--model-dir', type=Path)
|
| 69 |
+
source.add_argument('--endpoint')
|
| 70 |
+
parser.add_argument('--model', default='fev-4b')
|
| 71 |
+
parser.add_argument('--api-key-env', default='FEV_API_KEY')
|
| 72 |
+
parser.add_argument('--execution', choices=('reference', 'compiled'), default='reference')
|
| 73 |
+
parser.add_argument('--adaptation-dir', type=Path)
|
| 74 |
+
args = parser.parse_args()
|
| 75 |
+
records = load_records(args.data)
|
| 76 |
+
provenance = {'dataSha256': hashlib.sha256(args.data.read_bytes()).hexdigest(),
|
| 77 |
+
'startedUtc': datetime.now(timezone.utc).isoformat(), 'execution': args.execution,
|
| 78 |
+
'requestedModel': args.model}
|
| 79 |
+
if args.endpoint:
|
| 80 |
+
predict = endpoint_predictor(args.endpoint, args.model, os.environ.get(args.api_key_env))
|
| 81 |
+
provenance['endpoint'] = args.endpoint
|
| 82 |
+
else:
|
| 83 |
+
from .model import Fev4B
|
| 84 |
+
model = Fev4B(args.model_dir, execution=args.execution, adaptation_dir=args.adaptation_dir)
|
| 85 |
+
predict = model.decide
|
| 86 |
+
provenance['integrity'] = model.integrity
|
| 87 |
+
provenance['modelManifestSha256'] = hashlib.sha256((args.model_dir/'manifest.json').read_bytes()).hexdigest()
|
| 88 |
+
provenance['temperature'] = model.model.temperature
|
| 89 |
+
provenance['adaptation'] = model.adaptation
|
| 90 |
+
result = evaluate(records, predict, args.out, provenance=provenance)
|
| 91 |
+
print(json.dumps(result['overall'], indent=2))
|
| 92 |
+
if result['overall']['errors']:
|
| 93 |
+
sys.exit(1)
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
if __name__ == '__main__':
|
| 97 |
+
main()
|
fev4b/calibrate.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Fit one temperature on a separate calibration partition, writing a new overlay."""
|
| 2 |
+
import argparse
|
| 3 |
+
import json
|
| 4 |
+
import math
|
| 5 |
+
import shutil
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
from .adaptation import verify_adaptation
|
| 9 |
+
from .data import load_records, request_of, state_hash
|
| 10 |
+
from .metrics import rows_for, report_for
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def rescale(values, factor):
|
| 14 |
+
logits = [math.log(max(p, 1e-12))/factor for p in values]
|
| 15 |
+
maximum = max(logits)
|
| 16 |
+
exps = [math.exp(v-maximum) for v in logits]
|
| 17 |
+
return [v/sum(exps) for v in exps]
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def fit_temperature(rows):
|
| 21 |
+
if not rows or any(r['error'] is not None for r in rows):
|
| 22 |
+
raise ValueError('Calibration needs complete valid predictions')
|
| 23 |
+
def nll(factor):
|
| 24 |
+
return sum(-math.log(max(rescale(r['probabilities'], factor)[r['target']], 1e-12)) for r in rows)/len(rows)
|
| 25 |
+
# Include the unchanged temperature; minimize calibration NLL only.
|
| 26 |
+
factors = sorted({1., *(math.exp(-2 + i*.02) for i in range(201))})
|
| 27 |
+
best = min(factors, key=nll)
|
| 28 |
+
return best, {'beforeNll': nll(1.), 'afterNll': nll(best),
|
| 29 |
+
'factorBounds': [math.exp(-2), math.exp(2)],
|
| 30 |
+
'interpretation': 'In-sample calibration fit; independent test needed before adoption'}
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def main():
|
| 34 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 35 |
+
parser.add_argument('--model-dir', type=Path, required=True)
|
| 36 |
+
parser.add_argument('--adaptation-dir', type=Path, required=True)
|
| 37 |
+
parser.add_argument('--data', type=Path, required=True)
|
| 38 |
+
parser.add_argument('--out', type=Path, required=True)
|
| 39 |
+
args = parser.parse_args()
|
| 40 |
+
meta = verify_adaptation(args.model_dir, args.adaptation_dir)
|
| 41 |
+
records = load_records(args.data)
|
| 42 |
+
consumed = meta['consumed']
|
| 43 |
+
if ({r['id'] for r in records} & set(consumed['ids'])
|
| 44 |
+
or {state_hash(r) for r in records} & set(consumed['states'])
|
| 45 |
+
or {r['group'] for r in records if 'group' in r} & set(consumed['groups'])):
|
| 46 |
+
raise ValueError('Calibration overlaps training or development')
|
| 47 |
+
if args.out.exists():
|
| 48 |
+
raise ValueError('Calibration output already exists')
|
| 49 |
+
from .model import Fev4B
|
| 50 |
+
model = Fev4B(args.model_dir, adaptation_dir=args.adaptation_dir)
|
| 51 |
+
rows = [r for record in records for r in rows_for(record, model.decide(request_of(record)))]
|
| 52 |
+
factor, report = fit_temperature(rows)
|
| 53 |
+
args.out.mkdir(parents=True)
|
| 54 |
+
for name in meta['files']:
|
| 55 |
+
shutil.copyfile(args.adaptation_dir/name, args.out/name)
|
| 56 |
+
meta['temperature'] *= factor
|
| 57 |
+
meta['temperatureStatus'] = 'fitted on separate calibration data; test not read'
|
| 58 |
+
meta['consumed']['ids'] += [r['id'] for r in records]
|
| 59 |
+
meta['consumed']['states'] += [state_hash(r) for r in records]
|
| 60 |
+
meta['consumed']['groups'] += [r['group'] for r in records if 'group' in r]
|
| 61 |
+
(args.out/'adaptation.json').write_text(json.dumps(meta, indent=2)+'\n')
|
| 62 |
+
(args.out/'calibration-report.json').write_text(json.dumps({**report, 'factor': factor,
|
| 63 |
+
'calibrationMetricsBefore': report_for(rows), 'records': len(records)}, indent=2)+'\n')
|
| 64 |
+
print(json.dumps(report))
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
if __name__ == '__main__':
|
| 68 |
+
main()
|
fev4b/data.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Labelled typed requests and leakage-aware deterministic data splitting."""
|
| 2 |
+
import argparse
|
| 3 |
+
import hashlib
|
| 4 |
+
import json
|
| 5 |
+
import random
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
from .decisions import validate_request
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def canonical(value):
|
| 12 |
+
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(',', ':'), allow_nan=False)
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def state_hash(row):
|
| 16 |
+
return hashlib.sha256(canonical(row['state']).encode()).hexdigest()
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def request_of(row):
|
| 20 |
+
return validate_request({'state': row['state'], 'questions': {
|
| 21 |
+
key: {k: v for k, v in q.items() if k != 'label'} for key, q in row['questions'].items()}})
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def option_keys(question):
|
| 25 |
+
if question['type'] == 'choice':
|
| 26 |
+
return list(question['criteria'])
|
| 27 |
+
if question['type'] == 'noul':
|
| 28 |
+
return ['false', 'true']
|
| 29 |
+
return [str(i) for i in range(len(question['criteria']))]
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def target_index(question):
|
| 33 |
+
label = question['label']
|
| 34 |
+
if question['type'] == 'noul':
|
| 35 |
+
if type(label) is not bool:
|
| 36 |
+
raise ValueError('Noul label must be boolean')
|
| 37 |
+
return int(label)
|
| 38 |
+
if question['type'] == 'score':
|
| 39 |
+
if type(label) is not int or not 0 <= label < len(question['criteria']):
|
| 40 |
+
raise ValueError('Score label must be a zero-based level index')
|
| 41 |
+
return label
|
| 42 |
+
if not isinstance(label, str) or label not in question['criteria']:
|
| 43 |
+
raise ValueError('Choice label must name an option')
|
| 44 |
+
return list(question['criteria']).index(label)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def load_records(path):
|
| 48 |
+
records = []
|
| 49 |
+
seen = set()
|
| 50 |
+
for line in Path(path).read_text().splitlines():
|
| 51 |
+
if not line.strip():
|
| 52 |
+
continue
|
| 53 |
+
row = json.loads(line)
|
| 54 |
+
if (not isinstance(row, dict) or set(row) - {'id', 'group', 'domain', 'source', 'state', 'questions'}
|
| 55 |
+
or not isinstance(row.get('id'), str) or not row['id'] or row['id'] in seen):
|
| 56 |
+
raise ValueError('Records require unique nonempty IDs and supported fields')
|
| 57 |
+
for name in ('group', 'domain', 'source'):
|
| 58 |
+
if name in row and (not isinstance(row[name], str) or not row[name]):
|
| 59 |
+
raise ValueError('Record metadata must be nonempty strings')
|
| 60 |
+
request_of(row)
|
| 61 |
+
for question in row['questions'].values():
|
| 62 |
+
if set(question) - {'type', 'instructions', 'criteria', 'label'} or 'label' not in question:
|
| 63 |
+
raise ValueError('Every question needs a label')
|
| 64 |
+
target_index(question)
|
| 65 |
+
seen.add(row['id'])
|
| 66 |
+
records.append(row)
|
| 67 |
+
if not records:
|
| 68 |
+
raise ValueError('Dataset is empty')
|
| 69 |
+
return records
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def require_disjoint(*partitions):
|
| 73 |
+
prior_ids, prior_states, prior_groups = set(), set(), set()
|
| 74 |
+
for records in partitions:
|
| 75 |
+
ids = {r['id'] for r in records}
|
| 76 |
+
states = {state_hash(r) for r in records}
|
| 77 |
+
groups = {r['group'] for r in records if 'group' in r}
|
| 78 |
+
if ids & prior_ids or states & prior_states or groups & prior_groups:
|
| 79 |
+
raise ValueError('Partitions overlap by ID, exact state, or supplied group')
|
| 80 |
+
prior_ids |= ids
|
| 81 |
+
prior_states |= states
|
| 82 |
+
prior_groups |= groups
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def split_records(records, seed=42):
|
| 86 |
+
# Connected components prevent a repeated state from bridging two groups.
|
| 87 |
+
parent = list(range(len(records)))
|
| 88 |
+
def find(i):
|
| 89 |
+
while parent[i] != i:
|
| 90 |
+
parent[i] = parent[parent[i]]
|
| 91 |
+
i = parent[i]
|
| 92 |
+
return i
|
| 93 |
+
owners = {}
|
| 94 |
+
for i, row in enumerate(records):
|
| 95 |
+
keys = [('state', state_hash(row))]
|
| 96 |
+
if 'group' in row:
|
| 97 |
+
keys.append(('group', row['group']))
|
| 98 |
+
for key in keys:
|
| 99 |
+
if key in owners:
|
| 100 |
+
parent[find(i)] = find(owners[key])
|
| 101 |
+
owners[key] = i
|
| 102 |
+
groups = {}
|
| 103 |
+
for i, row in enumerate(records):
|
| 104 |
+
groups.setdefault(find(i), []).append(row)
|
| 105 |
+
values = list(groups.values())
|
| 106 |
+
if len(values) < 4:
|
| 107 |
+
raise ValueError('Need at least four independent groups for train/dev/calibration/test')
|
| 108 |
+
random.Random(seed).shuffle(values)
|
| 109 |
+
n = max(1, len(values) // 10)
|
| 110 |
+
boundaries = (len(values) - 3*n, len(values) - 2*n, len(values) - n, len(values))
|
| 111 |
+
result = {}
|
| 112 |
+
start = 0
|
| 113 |
+
for name, stop in zip(('train', 'development', 'calibration', 'test'), boundaries):
|
| 114 |
+
result[name] = [row for group in values[start:stop] for row in group]
|
| 115 |
+
start = stop
|
| 116 |
+
require_disjoint(*result.values())
|
| 117 |
+
return result
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def main():
|
| 121 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 122 |
+
parser.add_argument('--data', type=Path, required=True)
|
| 123 |
+
parser.add_argument('--out', type=Path, required=True)
|
| 124 |
+
parser.add_argument('--seed', type=int, default=42)
|
| 125 |
+
args = parser.parse_args()
|
| 126 |
+
records = load_records(args.data)
|
| 127 |
+
partitions = split_records(records, args.seed)
|
| 128 |
+
args.out.mkdir(parents=True, exist_ok=False)
|
| 129 |
+
manifest = {'seed': args.seed, 'sourceSha256': hashlib.sha256(args.data.read_bytes()).hexdigest(),
|
| 130 |
+
'grouping': 'connected exact states and supplied family groups; not fuzzy decontamination', 'files': {}}
|
| 131 |
+
for name, rows in partitions.items():
|
| 132 |
+
path = args.out / (name + '.jsonl')
|
| 133 |
+
path.write_text(''.join(canonical(row) + '\n' for row in rows))
|
| 134 |
+
manifest['files'][path.name] = {'records': len(rows), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()}
|
| 135 |
+
(args.out / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n')
|
| 136 |
+
print(json.dumps(manifest))
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
if __name__ == '__main__':
|
| 140 |
+
main()
|
fev4b/integrity.py
CHANGED
|
@@ -15,7 +15,7 @@ def file_sha256(path):
|
|
| 15 |
def verify_package(directory):
|
| 16 |
root = Path(directory).resolve()
|
| 17 |
manifest = json.loads((root / 'manifest.json').read_text())
|
| 18 |
-
if manifest['model'] != 'Fev-4B' or manifest['version'] not in ('0.1.0', '0.2.0'):
|
| 19 |
raise ValueError('Unsupported package manifest')
|
| 20 |
files = manifest.get('files')
|
| 21 |
if not isinstance(files, dict) or not files:
|
|
|
|
| 15 |
def verify_package(directory):
|
| 16 |
root = Path(directory).resolve()
|
| 17 |
manifest = json.loads((root / 'manifest.json').read_text())
|
| 18 |
+
if manifest['model'] != 'Fev-4B' or manifest['version'] not in ('0.1.0', '0.2.0', '0.2.1'):
|
| 19 |
raise ValueError('Unsupported package manifest')
|
| 20 |
files = manifest.get('files')
|
| 21 |
if not isinstance(files, dict) or not files:
|
fev4b/metrics.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Distribution metrics, with all failed questions retained in the denominator."""
|
| 2 |
+
import math
|
| 3 |
+
|
| 4 |
+
from .data import option_keys, target_index
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def distribution(question, answer):
|
| 8 |
+
if answer.get('type') != question['type']:
|
| 9 |
+
raise ValueError('Answer type mismatch')
|
| 10 |
+
keys = option_keys(question)
|
| 11 |
+
if question['type'] == 'noul':
|
| 12 |
+
p = answer['noul']
|
| 13 |
+
values = [1-p, p]
|
| 14 |
+
else:
|
| 15 |
+
if set(answer['probabilities']) != set(keys):
|
| 16 |
+
raise ValueError('Wrong probability keys')
|
| 17 |
+
values = [answer['probabilities'][k] for k in keys]
|
| 18 |
+
if (any(type(p) not in (int, float) or not math.isfinite(p) or not 0 <= p <= 1 for p in values)
|
| 19 |
+
or not math.isclose(sum(values), 1, abs_tol=.02)):
|
| 20 |
+
raise ValueError('Invalid distribution')
|
| 21 |
+
# Hosted APIs serialize rounded probabilities. Normalize only within the
|
| 22 |
+
# API tolerance; keep the original response in predictions.jsonl.
|
| 23 |
+
return [p/sum(values) for p in values]
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def rows_for(record, response=None, error=None):
|
| 27 |
+
rows = []
|
| 28 |
+
if response is not None and set(response['answers']) != set(record['questions']):
|
| 29 |
+
raise ValueError('Incomplete response')
|
| 30 |
+
for key, question in record['questions'].items():
|
| 31 |
+
row = {'id': record['id'], 'question': key, 'type': question['type'],
|
| 32 |
+
'source': record.get('source', record.get('domain', 'unspecified')), 'error': error}
|
| 33 |
+
if error is None:
|
| 34 |
+
values = distribution(question, response['answers'][key])
|
| 35 |
+
target = target_index(question)
|
| 36 |
+
predicted = max(range(len(values)), key=values.__getitem__)
|
| 37 |
+
row.update(correct=predicted == target, confidence=max(values),
|
| 38 |
+
probabilities=values, target=target,
|
| 39 |
+
brier=sum((p - int(i == target))**2 for i, p in enumerate(values)),
|
| 40 |
+
nll=-math.log(max(values[target], 1e-12)))
|
| 41 |
+
if question['type'] == 'score':
|
| 42 |
+
row['ordinalAbsoluteError'] = abs(sum(i*p for i, p in enumerate(values)) - target)
|
| 43 |
+
rows.append(row)
|
| 44 |
+
return rows
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def summarize(rows):
|
| 48 |
+
good = [r for r in rows if r['error'] is None]
|
| 49 |
+
bins = [[] for _ in range(10)]
|
| 50 |
+
for row in good:
|
| 51 |
+
bins[min(9, int(row['confidence']*10))].append(row)
|
| 52 |
+
ece = sum(len(b) * abs(sum(r['correct'] for r in b)/len(b) - sum(r['confidence'] for r in b)/len(b))
|
| 53 |
+
for b in bins if b) / len(good) if good else None
|
| 54 |
+
coverage = []
|
| 55 |
+
for threshold in (.5, .7, .8, .9, .95, .99):
|
| 56 |
+
accepted = [r for r in good if r['confidence'] >= threshold]
|
| 57 |
+
coverage.append({'threshold': threshold, 'accepted': len(accepted),
|
| 58 |
+
'coverage': len(accepted)/len(rows) if rows else 0,
|
| 59 |
+
'errorRate': sum(not r['correct'] for r in accepted)/len(accepted) if accepted else None})
|
| 60 |
+
return {'questions': len(rows), 'answered': len(good), 'errors': len(rows)-len(good),
|
| 61 |
+
'correct': sum(r['correct'] for r in good),
|
| 62 |
+
'accuracyAllQuestions': sum(r['correct'] for r in good)/len(rows) if rows else None,
|
| 63 |
+
'brierAnswered': sum(r['brier'] for r in good)/len(good) if good else None,
|
| 64 |
+
'nllAnswered': sum(r['nll'] for r in good)/len(good) if good else None,
|
| 65 |
+
'ece10Answered': ece,
|
| 66 |
+
'confidentWrongAt09': sum(not r['correct'] and r['confidence'] >= .9 for r in good),
|
| 67 |
+
'coverageByFixedThreshold': coverage,
|
| 68 |
+
'coverageInterpretation': 'Observed on this dataset; not a future error guarantee. Thresholds not tuned here.'}
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def report_for(rows):
|
| 72 |
+
return {'overall': summarize(rows),
|
| 73 |
+
'byType': {k: summarize([r for r in rows if r['type'] == k]) for k in sorted({r['type'] for r in rows})},
|
| 74 |
+
'bySource': {k: summarize([r for r in rows if r['source'] == k]) for k in sorted({r['source'] for r in rows})}}
|
fev4b/model.py
CHANGED
|
@@ -18,6 +18,7 @@ from fev.ranked_membership import resolve
|
|
| 18 |
from fev.research_controls import compare_responses
|
| 19 |
from .integrity import verify_package
|
| 20 |
from .decisions import decide_with_predict, validate_request
|
|
|
|
| 21 |
|
| 22 |
|
| 23 |
def route_with_predict(request, *, unknown_key, predict):
|
|
@@ -45,7 +46,7 @@ def route_with_predict(request, *, unknown_key, predict):
|
|
| 45 |
evidence.update({key: value['noul'] for key, value in secondary['answers'].items()})
|
| 46 |
decision = resolve(evidence, choice, config['precedence'], unknown_key=unknown_key, yes=.8, no=.2)
|
| 47 |
responses = [primary] + ([secondary] if secondary is not None else [])
|
| 48 |
-
return {'model': 'Fev-4B', 'version':
|
| 49 |
'route': decision.get('key'), 'probabilities': choice, 'applicability': evidence,
|
| 50 |
'queriedKeys': config['queriedKeys'], 'precedence': config['precedence'],
|
| 51 |
'usage': {key: sum(part['usage'][key] for part in responses)
|
|
@@ -59,7 +60,9 @@ class Fev4B:
|
|
| 59 |
One instance keeps weights resident. Calls are serialized because concurrency
|
| 60 |
and concurrent GPU-memory use have not been qualified for this release.
|
| 61 |
"""
|
| 62 |
-
def __init__(self, directory, *, device='cuda:0'):
|
|
|
|
|
|
|
| 63 |
self.root = Path(directory).resolve()
|
| 64 |
self.integrity = verify_package(self.root)
|
| 65 |
# All files are bundled. Runtime inference must not resolve Hub resources.
|
|
@@ -106,6 +109,19 @@ class Fev4B:
|
|
| 106 |
if len(loras) != 240 or sum(p.numel() for p in loras) != 10387456 or not all(p.dtype == torch.float32 for p in loras):
|
| 107 |
raise ValueError('Adapter shape or dtype differs from the benchmark')
|
| 108 |
self.model.backbone.eval()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
self.parameters = list(self.model.backbone.parameters()) + list(self.model.readout.parameters())
|
| 110 |
if any(p.requires_grad for p in self.parameters):
|
| 111 |
raise ValueError('Inference parameters must be frozen')
|
|
|
|
| 18 |
from fev.research_controls import compare_responses
|
| 19 |
from .integrity import verify_package
|
| 20 |
from .decisions import decide_with_predict, validate_request
|
| 21 |
+
from .version import VERSION
|
| 22 |
|
| 23 |
|
| 24 |
def route_with_predict(request, *, unknown_key, predict):
|
|
|
|
| 46 |
evidence.update({key: value['noul'] for key, value in secondary['answers'].items()})
|
| 47 |
decision = resolve(evidence, choice, config['precedence'], unknown_key=unknown_key, yes=.8, no=.2)
|
| 48 |
responses = [primary] + ([secondary] if secondary is not None else [])
|
| 49 |
+
return {'model': 'Fev-4B', 'version': VERSION, 'decision': decision,
|
| 50 |
'route': decision.get('key'), 'probabilities': choice, 'applicability': evidence,
|
| 51 |
'queriedKeys': config['queriedKeys'], 'precedence': config['precedence'],
|
| 52 |
'usage': {key: sum(part['usage'][key] for part in responses)
|
|
|
|
| 60 |
One instance keeps weights resident. Calls are serialized because concurrency
|
| 61 |
and concurrent GPU-memory use have not been qualified for this release.
|
| 62 |
"""
|
| 63 |
+
def __init__(self, directory, *, device='cuda:0', execution='reference', adaptation_dir=None):
|
| 64 |
+
if execution not in ('reference', 'compiled'):
|
| 65 |
+
raise ValueError('execution must be reference or compiled')
|
| 66 |
self.root = Path(directory).resolve()
|
| 67 |
self.integrity = verify_package(self.root)
|
| 68 |
# All files are bundled. Runtime inference must not resolve Hub resources.
|
|
|
|
| 109 |
if len(loras) != 240 or sum(p.numel() for p in loras) != 10387456 or not all(p.dtype == torch.float32 for p in loras):
|
| 110 |
raise ValueError('Adapter shape or dtype differs from the benchmark')
|
| 111 |
self.model.backbone.eval()
|
| 112 |
+
self.adaptation = None
|
| 113 |
+
if adaptation_dir is not None:
|
| 114 |
+
from .adaptation import load_adaptation
|
| 115 |
+
self.adaptation = load_adaptation(self, adaptation_dir)
|
| 116 |
+
# Construct after restoring the shared adapter. Graphs, FP8, pruning and
|
| 117 |
+
# prefetching were not qualified and are intentionally disabled.
|
| 118 |
+
if execution == 'compiled':
|
| 119 |
+
from pplx_decider.fast import FastPath
|
| 120 |
+
from pplx_decider.model import FastOptions
|
| 121 |
+
self.model.fast = FastPath(self.model, FastOptions(
|
| 122 |
+
compile='layer', cuda_graphs=False, prune_last=False,
|
| 123 |
+
fp8=False, prefetch=False, token_budget=16384))
|
| 124 |
+
self.execution = execution
|
| 125 |
self.parameters = list(self.model.backbone.parameters()) + list(self.model.readout.parameters())
|
| 126 |
if any(p.requires_grad for p in self.parameters):
|
| 127 |
raise ValueError('Inference parameters must be frozen')
|
fev4b/train.py
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Continue the released shared LoRA on generic labelled typed decisions (CUDA)."""
|
| 2 |
+
import argparse
|
| 3 |
+
import json
|
| 4 |
+
import math
|
| 5 |
+
import random
|
| 6 |
+
import shutil
|
| 7 |
+
import time
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
from .adaptation import base_identity
|
| 11 |
+
from .data import load_records, request_of, require_disjoint, target_index, state_hash
|
| 12 |
+
from .integrity import file_sha256
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def differentiable_logits(wrapper, request):
|
| 16 |
+
"""Same encoder, full attention and readout as reference inference, with gradients."""
|
| 17 |
+
from pplx_decider.protocol import left_pad
|
| 18 |
+
model = wrapper.model
|
| 19 |
+
chunks = model.encoder.plan(request, 16384)
|
| 20 |
+
if len(chunks) != 1 or chunks[0].length > 16384:
|
| 21 |
+
raise ValueError('Training request exceeds the native single-sequence context')
|
| 22 |
+
chunk = chunks[0]
|
| 23 |
+
inputs = {k: v.to(model.device) for k, v in left_pad(chunks, model.encoder.pad_id).items()}
|
| 24 |
+
hidden = model.backbone(**inputs, use_cache=False).last_hidden_state
|
| 25 |
+
offset = inputs['input_ids'].shape[1] - chunk.length
|
| 26 |
+
logits = model.readout(hidden[0, chunk.markers.to(model.device) + offset]).float()
|
| 27 |
+
return {key: logits[i, :count] for i, (key, count) in enumerate(zip(chunk.keys, chunk.counts, strict=True))}
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def train(model_dir, records, development, out, *, steps=100, learning_rate=2e-5, seed=42):
|
| 31 |
+
import torch
|
| 32 |
+
from peft import get_peft_model_state_dict
|
| 33 |
+
from safetensors.torch import save_file
|
| 34 |
+
from .model import Fev4B
|
| 35 |
+
from .benchmark import evaluate
|
| 36 |
+
if steps < 1 or not math.isfinite(learning_rate) or not 0 < learning_rate <= .001:
|
| 37 |
+
raise ValueError('Invalid training steps or learning rate')
|
| 38 |
+
require_disjoint(records, development)
|
| 39 |
+
domains = {row.get('domain') for row in records}
|
| 40 |
+
if None in domains or len(domains) < 2:
|
| 41 |
+
raise ValueError('Shared training requires domain metadata and at least two domains')
|
| 42 |
+
out = Path(out)
|
| 43 |
+
out.mkdir(parents=True, exist_ok=False)
|
| 44 |
+
torch.manual_seed(seed)
|
| 45 |
+
model = Fev4B(model_dir)
|
| 46 |
+
# The inference loader sets its own seed while constructing/restoring the
|
| 47 |
+
# released adapter. Training randomness belongs to this experiment.
|
| 48 |
+
torch.manual_seed(seed)
|
| 49 |
+
# Planning is checked before allocating optimizer state or taking a step.
|
| 50 |
+
for row in records + development:
|
| 51 |
+
chunks = model.model.encoder.plan(request_of(row), 16384)
|
| 52 |
+
if len(chunks) != 1 or chunks[0].length > 16384:
|
| 53 |
+
raise ValueError('Training/evaluation request exceeds context; nothing is truncated')
|
| 54 |
+
baseline = evaluate(development, model.decide, out/'baseline', provenance={'purpose': 'training development baseline'})
|
| 55 |
+
if baseline['overall']['errors']:
|
| 56 |
+
raise RuntimeError('Baseline evaluation failed; training was not started')
|
| 57 |
+
native = model.model
|
| 58 |
+
native.backbone.language_model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={'use_reentrant': False})
|
| 59 |
+
trainable = []
|
| 60 |
+
frozen = []
|
| 61 |
+
for name, parameter in native.backbone.named_parameters():
|
| 62 |
+
if 'lora_' in name:
|
| 63 |
+
parameter.requires_grad_(True)
|
| 64 |
+
trainable.append(parameter)
|
| 65 |
+
else:
|
| 66 |
+
frozen.append((parameter, parameter._version))
|
| 67 |
+
frozen += [(p, p._version) for p in native.readout.parameters()]
|
| 68 |
+
if len(trainable) != 240 or sum(p.numel() for p in trainable) != 10387456:
|
| 69 |
+
raise ValueError('Only the existing shared rank-8 adapter may be trained')
|
| 70 |
+
before = {k: v.detach().cpu().clone() for k, v in get_peft_model_state_dict(native.backbone.language_model).items()}
|
| 71 |
+
optimizer = torch.optim.AdamW(trainable, lr=learning_rate, weight_decay=0)
|
| 72 |
+
native.backbone.train()
|
| 73 |
+
order = list(range(len(records)))
|
| 74 |
+
rng = random.Random(seed)
|
| 75 |
+
history = []
|
| 76 |
+
started = time.perf_counter()
|
| 77 |
+
for step in range(steps):
|
| 78 |
+
if step % len(order) == 0:
|
| 79 |
+
rng.shuffle(order)
|
| 80 |
+
record = records[order[step % len(order)]]
|
| 81 |
+
optimizer.zero_grad(set_to_none=True)
|
| 82 |
+
values = differentiable_logits(model, request_of(record))
|
| 83 |
+
losses = [torch.nn.functional.cross_entropy(
|
| 84 |
+
(values[key]/native.temperature)[None, :],
|
| 85 |
+
torch.tensor([target_index(question)], device=native.device))
|
| 86 |
+
for key, question in record['questions'].items()]
|
| 87 |
+
loss = torch.stack(losses).mean()
|
| 88 |
+
if not torch.isfinite(loss):
|
| 89 |
+
raise RuntimeError('Non-finite training loss')
|
| 90 |
+
loss.backward()
|
| 91 |
+
gradients = [p.grad for p in trainable if p.grad is not None]
|
| 92 |
+
if not gradients or not all(torch.isfinite(g).all() for g in gradients):
|
| 93 |
+
raise RuntimeError('Missing or non-finite adapter gradients')
|
| 94 |
+
norm = torch.nn.utils.clip_grad_norm_(trainable, 1.0, error_if_nonfinite=True)
|
| 95 |
+
optimizer.step()
|
| 96 |
+
history.append({'step': step+1, 'loss': float(loss.detach()), 'gradientNorm': float(norm)})
|
| 97 |
+
print(json.dumps(history[-1]), flush=True)
|
| 98 |
+
if any(p._version != version or p.grad is not None for p, version in frozen):
|
| 99 |
+
raise RuntimeError('Frozen backbone/readout changed')
|
| 100 |
+
state = {k: v.detach().cpu().contiguous() for k, v in get_peft_model_state_dict(native.backbone.language_model).items()}
|
| 101 |
+
changed = sum(not torch.equal(v, before[k]) for k, v in state.items())
|
| 102 |
+
if not changed:
|
| 103 |
+
raise RuntimeError('Training did not update the adapter')
|
| 104 |
+
save_file(state, str(out/'adapter_model.safetensors'))
|
| 105 |
+
shutil.copyfile(Path(model_dir)/'adapter/adapter_config.json', out/'adapter_config.json')
|
| 106 |
+
meta = {'format': 'fev-adaptation-v1', 'baseIdentity': base_identity(model_dir),
|
| 107 |
+
'temperature': native.temperature, 'temperatureStatus': 'inherited; not fitted to these examples',
|
| 108 |
+
'files': {name: file_sha256(out/name) for name in ('adapter_config.json', 'adapter_model.safetensors')},
|
| 109 |
+
'purpose': 'shared customer-agnostic research adapter; not automatically adopted',
|
| 110 |
+
'steps': steps, 'seed': seed, 'learningRate': learning_rate,
|
| 111 |
+
'trainRecords': len(records), 'trainDomains': sorted(domains), 'changedAdapterTensors': changed,
|
| 112 |
+
'consumed': {'ids': sorted({r['id'] for r in records + development}),
|
| 113 |
+
'states': sorted({state_hash(r) for r in records + development}),
|
| 114 |
+
'groups': sorted({r['group'] for r in records + development if 'group' in r})},
|
| 115 |
+
'backboneAndReadoutFrozen': True, 'seconds': time.perf_counter()-started}
|
| 116 |
+
(out/'adaptation.json').write_text(json.dumps(meta, indent=2)+'\n')
|
| 117 |
+
(out/'steps.json').write_text(json.dumps(history, indent=2)+'\n')
|
| 118 |
+
native.backbone.language_model.gradient_checkpointing_disable()
|
| 119 |
+
native.backbone.eval()
|
| 120 |
+
for p in trainable:
|
| 121 |
+
p.requires_grad_(False)
|
| 122 |
+
model._versions = [(p, p._version) for p in model.parameters]
|
| 123 |
+
adapted = evaluate(development, model.decide, out/'adapted', provenance={'purpose': 'training development result'})
|
| 124 |
+
if adapted['overall']['errors']:
|
| 125 |
+
raise RuntimeError('Adapted evaluation failed; overlay is not qualified')
|
| 126 |
+
summary = {'baseline': baseline['overall'], 'adapted': adapted['overall'],
|
| 127 |
+
'training': meta, 'adopted': False, 'testPartitionRead': False}
|
| 128 |
+
(out/'training-report.json').write_text(json.dumps(summary, indent=2)+'\n')
|
| 129 |
+
return summary
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def main():
|
| 133 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 134 |
+
parser.add_argument('--model-dir', type=Path, required=True)
|
| 135 |
+
parser.add_argument('--train', type=Path, required=True)
|
| 136 |
+
parser.add_argument('--development', type=Path, required=True)
|
| 137 |
+
parser.add_argument('--out', type=Path, required=True)
|
| 138 |
+
parser.add_argument('--steps', type=int, default=100)
|
| 139 |
+
parser.add_argument('--learning-rate', type=float, default=2e-5)
|
| 140 |
+
parser.add_argument('--seed', type=int, default=42)
|
| 141 |
+
args = parser.parse_args()
|
| 142 |
+
result = train(args.model_dir, load_records(args.train), load_records(args.development), args.out,
|
| 143 |
+
steps=args.steps, learning_rate=args.learning_rate, seed=args.seed)
|
| 144 |
+
result['dataHashes'] = {name: file_sha256(path) for name, path in
|
| 145 |
+
[('train', args.train), ('development', args.development)]}
|
| 146 |
+
(args.out/'training-report.json').write_text(json.dumps(result, indent=2)+'\n')
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
if __name__ == '__main__':
|
| 150 |
+
main()
|
fev4b/version.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
VERSION = '0.2.1'
|
fev_config.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"model": "Fev-4B",
|
| 3 |
-
"version": "0.2.
|
| 4 |
"runtime": {
|
| 5 |
"maxLength": 16384,
|
| 6 |
"batchTokens": 16384,
|
|
|
|
| 1 |
{
|
| 2 |
"model": "Fev-4B",
|
| 3 |
+
"version": "0.2.1",
|
| 4 |
"runtime": {
|
| 5 |
"maxLength": 16384,
|
| 6 |
"batchTokens": 16384,
|
manifest.json
CHANGED
|
@@ -1,34 +1,50 @@
|
|
| 1 |
{
|
| 2 |
"model": "Fev-4B",
|
| 3 |
-
"version": "0.2.
|
| 4 |
"files": {
|
| 5 |
".gitattributes": {
|
| 6 |
-
"bytes":
|
| 7 |
-
"sha256": "
|
| 8 |
},
|
| 9 |
".gitignore": {
|
| 10 |
-
"bytes":
|
| 11 |
-
"sha256": "
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
},
|
| 13 |
"CHANGES.md": {
|
| 14 |
-
"bytes":
|
| 15 |
-
"sha256": "
|
| 16 |
},
|
| 17 |
"LICENSE": {
|
| 18 |
"bytes": 11544,
|
| 19 |
"sha256": "bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a"
|
| 20 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
"NOTICE": {
|
| 22 |
"bytes": 132,
|
| 23 |
"sha256": "381fb81e814716a5ef8e5eae26a08607d052cf12f019dba9da9e81298ef5cb75"
|
| 24 |
},
|
| 25 |
"README.md": {
|
| 26 |
-
"bytes":
|
| 27 |
-
"sha256": "
|
| 28 |
},
|
| 29 |
"RUNNING.md": {
|
| 30 |
-
"bytes":
|
| 31 |
-
"sha256": "
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
},
|
| 33 |
"adapter/adapter_config.json": {
|
| 34 |
"bytes": 1136,
|
|
@@ -126,6 +142,66 @@
|
|
| 126 |
"bytes": 1268,
|
| 127 |
"sha256": "188cf0b9db5cfd6e92f1b63c646f5975be6f8383c6c9a2fa5dc2b48a8b229402"
|
| 128 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 129 |
"evaluation/summary.json": {
|
| 130 |
"bytes": 3598,
|
| 131 |
"sha256": "8529bfbb115f2b3b877efb2b3cdecfce728b562d4a209e803a245529faff4b21"
|
|
@@ -134,6 +210,14 @@
|
|
| 134 |
"bytes": 1342,
|
| 135 |
"sha256": "50ba21d99dfce090ad503edc9ee4920a77d4bc3d63f9933fb71a4acb093f0baa"
|
| 136 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 137 |
"fev/__init__.py": {
|
| 138 |
"bytes": 0,
|
| 139 |
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
|
@@ -166,61 +250,57 @@
|
|
| 166 |
"bytes": 273,
|
| 167 |
"sha256": "d4f2e7ee272979f3a1e1c9e04b9815cbe254585a75b465089a9053a813414795"
|
| 168 |
},
|
| 169 |
-
"fev4b/
|
| 170 |
-
"bytes":
|
| 171 |
-
"sha256": "
|
| 172 |
},
|
| 173 |
-
"fev4b/
|
| 174 |
-
"bytes":
|
| 175 |
-
"sha256": "
|
| 176 |
-
},
|
| 177 |
-
"fev_config.json": {
|
| 178 |
-
"bytes": 243,
|
| 179 |
-
"sha256": "f72bdcf114d213a17cd77c4e24608e5c37c39f12f7e6f3b3a09d417f7fbddf8c"
|
| 180 |
},
|
| 181 |
-
"
|
| 182 |
-
"bytes":
|
| 183 |
-
"sha256": "
|
| 184 |
},
|
| 185 |
-
"
|
| 186 |
-
"bytes":
|
| 187 |
-
"sha256": "
|
| 188 |
},
|
| 189 |
-
"
|
| 190 |
-
"bytes":
|
| 191 |
-
"sha256": "
|
| 192 |
},
|
| 193 |
-
"
|
| 194 |
-
"bytes":
|
| 195 |
-
"sha256": "
|
| 196 |
},
|
| 197 |
-
"
|
| 198 |
-
"bytes":
|
| 199 |
-
"sha256": "
|
| 200 |
},
|
| 201 |
-
"
|
| 202 |
-
"bytes":
|
| 203 |
-
"sha256": "
|
| 204 |
},
|
| 205 |
-
"
|
| 206 |
-
"bytes":
|
| 207 |
-
"sha256": "
|
| 208 |
},
|
| 209 |
-
"
|
| 210 |
-
"bytes":
|
| 211 |
-
"sha256": "
|
| 212 |
},
|
| 213 |
-
"
|
| 214 |
-
"bytes":
|
| 215 |
-
"sha256": "
|
| 216 |
},
|
| 217 |
-
"
|
| 218 |
-
"bytes":
|
| 219 |
-
"sha256": "
|
| 220 |
},
|
| 221 |
-
"
|
| 222 |
-
"bytes":
|
| 223 |
-
"sha256": "
|
| 224 |
},
|
| 225 |
"requirements-serve.txt": {
|
| 226 |
"bytes": 18,
|
|
@@ -230,17 +310,61 @@
|
|
| 230 |
"bytes": 52,
|
| 231 |
"sha256": "408006931c14be75c13e7cd0b4215fc92c229aeee567912032a4ce33a2637d8e"
|
| 232 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 233 |
"serve.py": {
|
| 234 |
-
"bytes":
|
| 235 |
-
"sha256": "
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 236 |
},
|
| 237 |
"tests/test_decisions.py": {
|
| 238 |
"bytes": 6724,
|
| 239 |
"sha256": "8ed5a03d65da031d00cce6c9eef2c062bec19f58c96dbdeed30f01d33e7e6b03"
|
| 240 |
},
|
| 241 |
-
"
|
| 242 |
-
"bytes":
|
| 243 |
-
"sha256": "
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 244 |
}
|
| 245 |
}
|
| 246 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"model": "Fev-4B",
|
| 3 |
+
"version": "0.2.1",
|
| 4 |
"files": {
|
| 5 |
".gitattributes": {
|
| 6 |
+
"bytes": 35,
|
| 7 |
+
"sha256": "f5c48f8d4d2d2607912400d98e52eba057418b8dfbbc5ceeaf66cfaac33664b4"
|
| 8 |
},
|
| 9 |
".gitignore": {
|
| 10 |
+
"bytes": 126,
|
| 11 |
+
"sha256": "588d2dfddaee6a14b0692f4d3fb213f0da06f3e7df5606678ae430c1da484cb5"
|
| 12 |
+
},
|
| 13 |
+
"API.md": {
|
| 14 |
+
"bytes": 4224,
|
| 15 |
+
"sha256": "16369fd2f3b8671c22f61680a6bef2fe60108fc458d55c0e4c10b8cf51363c62"
|
| 16 |
+
},
|
| 17 |
+
"BENCHMARKS.md": {
|
| 18 |
+
"bytes": 9808,
|
| 19 |
+
"sha256": "515d1ec216212b2ec4e49b4c7e4a48019a87ba430974c435d1852e5606fcbb16"
|
| 20 |
},
|
| 21 |
"CHANGES.md": {
|
| 22 |
+
"bytes": 1819,
|
| 23 |
+
"sha256": "e55f12b6bf318ff1bd8507e9e8725bb5a06f84257c20efe406987b4dd270210c"
|
| 24 |
},
|
| 25 |
"LICENSE": {
|
| 26 |
"bytes": 11544,
|
| 27 |
"sha256": "bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a"
|
| 28 |
},
|
| 29 |
+
"MODEL_CARD.md": {
|
| 30 |
+
"bytes": 11647,
|
| 31 |
+
"sha256": "216aed02e370fba7c7f9cb147d8e4fd3d7378a9003643c4ef7b70f38280d4cd5"
|
| 32 |
+
},
|
| 33 |
"NOTICE": {
|
| 34 |
"bytes": 132,
|
| 35 |
"sha256": "381fb81e814716a5ef8e5eae26a08607d052cf12f019dba9da9e81298ef5cb75"
|
| 36 |
},
|
| 37 |
"README.md": {
|
| 38 |
+
"bytes": 11647,
|
| 39 |
+
"sha256": "216aed02e370fba7c7f9cb147d8e4fd3d7378a9003643c4ef7b70f38280d4cd5"
|
| 40 |
},
|
| 41 |
"RUNNING.md": {
|
| 42 |
+
"bytes": 7334,
|
| 43 |
+
"sha256": "c7ec72f2f282589ad3be4328cc93d04c13fd7c1a7df791c95a5d4740aeaa6c29"
|
| 44 |
+
},
|
| 45 |
+
"TRAINING.md": {
|
| 46 |
+
"bytes": 580,
|
| 47 |
+
"sha256": "4a7dfa55a8e8203ea800644ff3a510d6abf9aea54001ab9d3bb04fa9f272991a"
|
| 48 |
},
|
| 49 |
"adapter/adapter_config.json": {
|
| 50 |
"bytes": 1136,
|
|
|
|
| 142 |
"bytes": 1268,
|
| 143 |
"sha256": "188cf0b9db5cfd6e92f1b63c646f5975be6f8383c6c9a2fa5dc2b48a8b229402"
|
| 144 |
},
|
| 145 |
+
"evaluation/api-verification.json": {
|
| 146 |
+
"bytes": 2579,
|
| 147 |
+
"sha256": "05c3571281394c374018240552f3d357cf77fb38b62bfa70c4698d5aae130094"
|
| 148 |
+
},
|
| 149 |
+
"evaluation/benchmarks/fev/predictions.jsonl": {
|
| 150 |
+
"bytes": 286434,
|
| 151 |
+
"sha256": "cb8867ca4ca92c8885b6ba7543b98c0422c8461236b0cf24a8a09d6436d39b41"
|
| 152 |
+
},
|
| 153 |
+
"evaluation/benchmarks/fev/report.json": {
|
| 154 |
+
"bytes": 18238,
|
| 155 |
+
"sha256": "73c194b6dfbc8f8acd2da3ce7a70c4ec03a2d87ecb001d71e24db1bad6a969dd"
|
| 156 |
+
},
|
| 157 |
+
"evaluation/benchmarks/fev/rows.json": {
|
| 158 |
+
"bytes": 360571,
|
| 159 |
+
"sha256": "3a2e05ae1316e47a62f2ad9536e1d9f5278057aa0bd17feef911eda4127e01d3"
|
| 160 |
+
},
|
| 161 |
+
"evaluation/benchmarks/jev/predictions.jsonl": {
|
| 162 |
+
"bytes": 248861,
|
| 163 |
+
"sha256": "a785a89be12dd282fa871d8a91e15b642253df1f9ebc5c910b1110d6705bef25"
|
| 164 |
+
},
|
| 165 |
+
"evaluation/benchmarks/jev/report.json": {
|
| 166 |
+
"bytes": 18448,
|
| 167 |
+
"sha256": "99b49c99a912a3e493a1a978ba0b49b560d29245e4cc0971d921a28ec71678e9"
|
| 168 |
+
},
|
| 169 |
+
"evaluation/benchmarks/jev/rows.json": {
|
| 170 |
+
"bytes": 292803,
|
| 171 |
+
"sha256": "62ab7b1f662fd48f10b295c854a5372d2b51d0c129f936842420b135c898f085"
|
| 172 |
+
},
|
| 173 |
+
"evaluation/benchmarks/kev/predictions.jsonl": {
|
| 174 |
+
"bytes": 227373,
|
| 175 |
+
"sha256": "4e3a1b5265312950ce964a7ef5d1d5c51528a2acc2b290a06d28898d804cd3bc"
|
| 176 |
+
},
|
| 177 |
+
"evaluation/benchmarks/kev/report.json": {
|
| 178 |
+
"bytes": 18461,
|
| 179 |
+
"sha256": "ed44fbbaf9dbe902920b90eb72dfad2480db9bdc2f43377c2cc813917e2158a7"
|
| 180 |
+
},
|
| 181 |
+
"evaluation/benchmarks/kev/rows.json": {
|
| 182 |
+
"bytes": 359621,
|
| 183 |
+
"sha256": "8c1a1a15833bdcbf2c275c904c0041b29bdc98ba568404bda9f88533c6d5b67e"
|
| 184 |
+
},
|
| 185 |
+
"evaluation/benchmarks/results.json": {
|
| 186 |
+
"bytes": 72127,
|
| 187 |
+
"sha256": "934b760401abd85e99e44e84efda402a7caaffc31191b466cd91469a833ff592"
|
| 188 |
+
},
|
| 189 |
+
"evaluation/benchmarks/runtime.json": {
|
| 190 |
+
"bytes": 1790,
|
| 191 |
+
"sha256": "3444077e4ae2cf17abb73ea09e32d8be0ad9f55b260e8cd1c7e26e5307b0516a"
|
| 192 |
+
},
|
| 193 |
+
"evaluation/benchmarks/russian-typed.jsonl": {
|
| 194 |
+
"bytes": 42621,
|
| 195 |
+
"sha256": "9ef2a76a365b0e1d582a9b2bceba23fd541389d7d5d4613142fe5b0685ec34eb"
|
| 196 |
+
},
|
| 197 |
+
"evaluation/benchmarks/training-validation.json": {
|
| 198 |
+
"bytes": 5779,
|
| 199 |
+
"sha256": "f62afd3d12bccf62e172f0ec19d37e042ed2d852b12dd371701a8ba36f0776f0"
|
| 200 |
+
},
|
| 201 |
+
"evaluation/packaging-verification.json": {
|
| 202 |
+
"bytes": 1855,
|
| 203 |
+
"sha256": "6abacf62c38fc5c5950ef26a207b7db5355655d92684c9eb44c1b12e6eb4434c"
|
| 204 |
+
},
|
| 205 |
"evaluation/summary.json": {
|
| 206 |
"bytes": 3598,
|
| 207 |
"sha256": "8529bfbb115f2b3b877efb2b3cdecfce728b562d4a209e803a245529faff4b21"
|
|
|
|
| 210 |
"bytes": 1342,
|
| 211 |
"sha256": "50ba21d99dfce090ad503edc9ee4920a77d4bc3d63f9933fb71a4acb093f0baa"
|
| 212 |
},
|
| 213 |
+
"examples/training.jsonl": {
|
| 214 |
+
"bytes": 32348,
|
| 215 |
+
"sha256": "92b4de1a87aeb2710b9ea7f50933b8352842481ef956996bd1d1c88ac10a4f6b"
|
| 216 |
+
},
|
| 217 |
+
"examples/typed-request.json": {
|
| 218 |
+
"bytes": 871,
|
| 219 |
+
"sha256": "f2ddf12d2029db1ca6aab8fed26626b913979755a51a9beec914aff8c9877823"
|
| 220 |
+
},
|
| 221 |
"fev/__init__.py": {
|
| 222 |
"bytes": 0,
|
| 223 |
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
|
|
|
| 250 |
"bytes": 273,
|
| 251 |
"sha256": "d4f2e7ee272979f3a1e1c9e04b9815cbe254585a75b465089a9053a813414795"
|
| 252 |
},
|
| 253 |
+
"fev4b/adaptation.py": {
|
| 254 |
+
"bytes": 2613,
|
| 255 |
+
"sha256": "046f40ab65cc631d34f8779a37344501d2263fea722b7361bdc651b828307af2"
|
| 256 |
},
|
| 257 |
+
"fev4b/benchmark.py": {
|
| 258 |
+
"bytes": 4658,
|
| 259 |
+
"sha256": "5d638598bb9e5e01ff8b74bb2c3f1702a532539896b38b2eb08f5b9688be718d"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 260 |
},
|
| 261 |
+
"fev4b/calibrate.py": {
|
| 262 |
+
"bytes": 3124,
|
| 263 |
+
"sha256": "008c5e0731fc8dd363b4ebc145eb34545c899cb770d85db284d71df6ac55ae18"
|
| 264 |
},
|
| 265 |
+
"fev4b/data.py": {
|
| 266 |
+
"bytes": 5489,
|
| 267 |
+
"sha256": "d36331206cb94c23e56de9a9fe2922f9842b235a968bac7bcdaaea62e4e2cc34"
|
| 268 |
},
|
| 269 |
+
"fev4b/decisions.py": {
|
| 270 |
+
"bytes": 5456,
|
| 271 |
+
"sha256": "b2122283d3be1afb4735ce353f091c7646b45280b7eeec08072cb4b79a3c7434"
|
| 272 |
},
|
| 273 |
+
"fev4b/integrity.py": {
|
| 274 |
+
"bytes": 1496,
|
| 275 |
+
"sha256": "30e4c14aa2fbc28a47252a8427179a8260f514b58c4f734debd84c91680535a7"
|
| 276 |
},
|
| 277 |
+
"fev4b/metrics.py": {
|
| 278 |
+
"bytes": 3886,
|
| 279 |
+
"sha256": "f9eb49549c5b66ac2404df5a397090400b172659c14e00da833f0efef38b2042"
|
| 280 |
},
|
| 281 |
+
"fev4b/model.py": {
|
| 282 |
+
"bytes": 9307,
|
| 283 |
+
"sha256": "aa63b29bcfeb5b1484f6eb8c2a486dae0f4c078405b7ca00ed54e9aa0cfa804e"
|
| 284 |
},
|
| 285 |
+
"fev4b/server.py": {
|
| 286 |
+
"bytes": 5384,
|
| 287 |
+
"sha256": "0822c2cadb9634801ce31ef080bd1faf6f442de03996c280fd1c76e3ce56aabc"
|
| 288 |
},
|
| 289 |
+
"fev4b/train.py": {
|
| 290 |
+
"bytes": 8295,
|
| 291 |
+
"sha256": "a4baf8cd3c7b785c07406ab14564911fe2761c804d7094f7edd1c48d186a5f02"
|
| 292 |
},
|
| 293 |
+
"fev4b/version.py": {
|
| 294 |
+
"bytes": 18,
|
| 295 |
+
"sha256": "59b34844cb8d885d9f8ed5a9a47df3b86ef29156922e72229ee537a8ad2102a0"
|
| 296 |
},
|
| 297 |
+
"fev_config.json": {
|
| 298 |
+
"bytes": 243,
|
| 299 |
+
"sha256": "82d8e43a188cfccafa21d40a0faa50e44e0d4b9cc0bb56e7e3a6fe2a343e4926"
|
| 300 |
},
|
| 301 |
+
"provenance.json": {
|
| 302 |
+
"bytes": 2291,
|
| 303 |
+
"sha256": "d47ec6ff8ac97f58fa486a3d42dd381894705b1d6c06c1a1d853201369de76f7"
|
| 304 |
},
|
| 305 |
"requirements-serve.txt": {
|
| 306 |
"bytes": 18,
|
|
|
|
| 310 |
"bytes": 52,
|
| 311 |
"sha256": "408006931c14be75c13e7cd0b4215fc92c229aeee567912032a4ce33a2637d8e"
|
| 312 |
},
|
| 313 |
+
"requirements.txt": {
|
| 314 |
+
"bytes": 189,
|
| 315 |
+
"sha256": "05a9befe0fa538b33ee3eb1639075449fa2104e33184c1c1425f63dd847dda68"
|
| 316 |
+
},
|
| 317 |
+
"run.py": {
|
| 318 |
+
"bytes": 1593,
|
| 319 |
+
"sha256": "3d32513dae741f18ba6fe0262fbd5cf2566e025e775da95cd94734e4f356225b"
|
| 320 |
+
},
|
| 321 |
"serve.py": {
|
| 322 |
+
"bytes": 1353,
|
| 323 |
+
"sha256": "7c63131c92ada0f2e278dc2c6f4f75c793fa61b7faadff31e1f375b2b028308e"
|
| 324 |
+
},
|
| 325 |
+
"skills/fev-finetune/SKILL.md": {
|
| 326 |
+
"bytes": 2464,
|
| 327 |
+
"sha256": "be1390b03002ef35a407f3df0da4206904da38ccc27038c62992abc55080a7be"
|
| 328 |
+
},
|
| 329 |
+
"skills/fev-finetune/references/training.md": {
|
| 330 |
+
"bytes": 8187,
|
| 331 |
+
"sha256": "0ba4f7b8c6986bc60f44e32f1bb17a4742b5634398f7a10f4ef9ffcef6f7ae8f"
|
| 332 |
},
|
| 333 |
"tests/test_decisions.py": {
|
| 334 |
"bytes": 6724,
|
| 335 |
"sha256": "8ed5a03d65da031d00cce6c9eef2c062bec19f58c96dbdeed30f01d33e7e6b03"
|
| 336 |
},
|
| 337 |
+
"tests/test_runtime.py": {
|
| 338 |
+
"bytes": 3101,
|
| 339 |
+
"sha256": "c32c8bb32b82e3a49b0b7d29e5ab78b9cc0dcfb3c347412927b1fe9870a89f78"
|
| 340 |
+
},
|
| 341 |
+
"tests/test_training_tools.py": {
|
| 342 |
+
"bytes": 5471,
|
| 343 |
+
"sha256": "ef3aeda2d526eefc85121c0d9436f65e8e942ea0421a7b0ddc4a490c803af4fc"
|
| 344 |
+
},
|
| 345 |
+
"tools/benchmark_kev.py": {
|
| 346 |
+
"bytes": 3049,
|
| 347 |
+
"sha256": "05ffde30813006e6bd6740122884c1eed0f1a93e691b42862308c83938614e4f"
|
| 348 |
+
},
|
| 349 |
+
"tools/compare_runs.py": {
|
| 350 |
+
"bytes": 3928,
|
| 351 |
+
"sha256": "65139f3c7aa4700642a9f4b0ab03a034b11512637f0d5f60c075dff476b1f8ef"
|
| 352 |
+
},
|
| 353 |
+
"tools/prepare_benchmark.py": {
|
| 354 |
+
"bytes": 2327,
|
| 355 |
+
"sha256": "a225ef7d944f91e149e7cf96dceb9e08709a2e92f4a97df1d19d119ae7e3279b"
|
| 356 |
+
},
|
| 357 |
+
"tools/train_hf.py": {
|
| 358 |
+
"bytes": 3066,
|
| 359 |
+
"sha256": "9865945aebb02191c45aced5d8573aab3a4f4201a06ff5896b56a05c4f6ab78e"
|
| 360 |
+
},
|
| 361 |
+
"checkpoint/model-00001-of-00002.safetensors": {
|
| 362 |
+
"bytes": 4972946624,
|
| 363 |
+
"sha256": "1c1a0c1ab331c7aec1b4b7932eabf80593b56c694fe2b779209c0e0413df1cef"
|
| 364 |
+
},
|
| 365 |
+
"checkpoint/model-00002-of-00002.safetensors": {
|
| 366 |
+
"bytes": 4105669472,
|
| 367 |
+
"sha256": "21ba8301660e066f5849b60b1a36a242fc54ca18dd9bd526fdcbc50889171b70"
|
| 368 |
}
|
| 369 |
}
|
| 370 |
}
|
provenance.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"model": "Fev-4B",
|
| 3 |
-
"version": "0.2.
|
| 4 |
"publisher": "murabcd",
|
| 5 |
"baseCheckpoint": {
|
| 6 |
"repo": "perplexity-ai/pplx-decider-flash-v1.2",
|
|
@@ -38,5 +38,18 @@
|
|
| 38 |
"sourceFrozenManifestSha256": "28652bb88f086b2be4c6251dda581f5bdfdf4466ec09bf862e6eca005dab0cbf",
|
| 39 |
"upstreamManifestNote": "Upstream README hash was stale; execution artifacts all verified. Package manifest records the actual included README hash.",
|
| 40 |
"weightsUnchangedSinceVersion": "0.1.0",
|
| 41 |
-
"sourceRepository": "https://github.com/murabcd/fev"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"model": "Fev-4B",
|
| 3 |
+
"version": "0.2.1",
|
| 4 |
"publisher": "murabcd",
|
| 5 |
"baseCheckpoint": {
|
| 6 |
"repo": "perplexity-ai/pplx-decider-flash-v1.2",
|
|
|
|
| 38 |
"sourceFrozenManifestSha256": "28652bb88f086b2be4c6251dda581f5bdfdf4466ec09bf862e6eca005dab0cbf",
|
| 39 |
"upstreamManifestNote": "Upstream README hash was stale; execution artifacts all verified. Package manifest records the actual included README hash.",
|
| 40 |
"weightsUnchangedSinceVersion": "0.1.0",
|
| 41 |
+
"sourceRepository": "https://github.com/murabcd/fev",
|
| 42 |
+
"releaseStatus": "released",
|
| 43 |
+
"executionModes": {
|
| 44 |
+
"default": "reference",
|
| 45 |
+
"optional": "compiled",
|
| 46 |
+
"cudaGraphs": false,
|
| 47 |
+
"fp8": false,
|
| 48 |
+
"readoutPruning": false,
|
| 49 |
+
"prefetch": false
|
| 50 |
+
},
|
| 51 |
+
"researchTraining": {
|
| 52 |
+
"tooling": "shared rank-8 decision cross-entropy; explicit separate overlay",
|
| 53 |
+
"adoptedIntoReleasedWeights": false
|
| 54 |
+
}
|
| 55 |
}
|
run.py
CHANGED
|
@@ -12,6 +12,8 @@ def main():
|
|
| 12 |
parser.add_argument('--model-dir', type=Path, default=Path(__file__).resolve().parent)
|
| 13 |
parser.add_argument('--input', type=Path, help='JSON containing request and unknownKey; default: stdin')
|
| 14 |
parser.add_argument('--device', default='cuda:0')
|
|
|
|
|
|
|
| 15 |
parser.add_argument('--verify-only', action='store_true', help='Verify all hashes without importing PyTorch')
|
| 16 |
args = parser.parse_args()
|
| 17 |
if args.verify_only:
|
|
@@ -20,7 +22,8 @@ def main():
|
|
| 20 |
payload = json.loads(args.input.read_text() if args.input else sys.stdin.read())
|
| 21 |
if not isinstance(payload, dict) or set(payload) != {'request', 'unknownKey'}:
|
| 22 |
raise ValueError('Expected exactly request and unknownKey')
|
| 23 |
-
model = Fev4B(args.model_dir, device=args.device
|
|
|
|
| 24 |
result = model.route(payload['request'], unknown_key=payload['unknownKey'])
|
| 25 |
# Raw native evidence is available through the Python interface for debugging.
|
| 26 |
result.pop('native')
|
|
|
|
| 12 |
parser.add_argument('--model-dir', type=Path, default=Path(__file__).resolve().parent)
|
| 13 |
parser.add_argument('--input', type=Path, help='JSON containing request and unknownKey; default: stdin')
|
| 14 |
parser.add_argument('--device', default='cuda:0')
|
| 15 |
+
parser.add_argument('--execution', choices=('reference', 'compiled'), default='reference')
|
| 16 |
+
parser.add_argument('--adaptation-dir', type=Path)
|
| 17 |
parser.add_argument('--verify-only', action='store_true', help='Verify all hashes without importing PyTorch')
|
| 18 |
args = parser.parse_args()
|
| 19 |
if args.verify_only:
|
|
|
|
| 22 |
payload = json.loads(args.input.read_text() if args.input else sys.stdin.read())
|
| 23 |
if not isinstance(payload, dict) or set(payload) != {'request', 'unknownKey'}:
|
| 24 |
raise ValueError('Expected exactly request and unknownKey')
|
| 25 |
+
model = Fev4B(args.model_dir, device=args.device, execution=args.execution,
|
| 26 |
+
adaptation_dir=args.adaptation_dir)
|
| 27 |
result = model.route(payload['request'], unknown_key=payload['unknownKey'])
|
| 28 |
# Raw native evidence is available through the Python interface for debugging.
|
| 29 |
result.pop('native')
|
serve.py
CHANGED
|
@@ -13,11 +13,14 @@ def main():
|
|
| 13 |
parser.add_argument('--host', default='127.0.0.1')
|
| 14 |
parser.add_argument('--port', type=int, default=8000)
|
| 15 |
parser.add_argument('--device', default='cuda:0')
|
|
|
|
|
|
|
| 16 |
args = parser.parse_args()
|
| 17 |
api_key = os.environ.get('FEV_API_KEY')
|
| 18 |
if args.host not in ('127.0.0.1', 'localhost') and not api_key:
|
| 19 |
parser.error('Set FEV_API_KEY before binding outside localhost')
|
| 20 |
-
model = Fev4B(args.model_dir, device=args.device
|
|
|
|
| 21 |
with create_server(model, (args.host, args.port), api_key=api_key) as server:
|
| 22 |
print(f'Fev ready at http://{args.host}:{args.port}', flush=True)
|
| 23 |
try:
|
|
|
|
| 13 |
parser.add_argument('--host', default='127.0.0.1')
|
| 14 |
parser.add_argument('--port', type=int, default=8000)
|
| 15 |
parser.add_argument('--device', default='cuda:0')
|
| 16 |
+
parser.add_argument('--execution', choices=('reference', 'compiled'), default='reference')
|
| 17 |
+
parser.add_argument('--adaptation-dir', type=Path, help='Explicit, verified research adapter overlay')
|
| 18 |
args = parser.parse_args()
|
| 19 |
api_key = os.environ.get('FEV_API_KEY')
|
| 20 |
if args.host not in ('127.0.0.1', 'localhost') and not api_key:
|
| 21 |
parser.error('Set FEV_API_KEY before binding outside localhost')
|
| 22 |
+
model = Fev4B(args.model_dir, device=args.device, execution=args.execution,
|
| 23 |
+
adaptation_dir=args.adaptation_dir)
|
| 24 |
with create_server(model, (args.host, args.port), api_key=api_key) as server:
|
| 25 |
print(f'Fev ready at http://{args.host}:{args.port}', flush=True)
|
| 26 |
try:
|
skills/fev-finetune/SKILL.md
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
name: fev-finetune
|
| 3 |
+
description: Train and evaluate a shared Fev-4B decision adapter, including labelled Choice/Noul/Score data, leakage-aware splits, temperature calibration, and bounded GPU jobs. Use for Fev training or adaptation, rather than configuring customer routing categories.
|
| 4 |
+
---
|
| 5 |
+
|
| 6 |
+
# Fev fine-tuning
|
| 7 |
+
|
| 8 |
+
Fev scores typed questions; it does not generate chat completions. Train its existing shared rank-8 adapter with native decision cross-entropy. Do not use a generic causal-LM SFT recipe or replace the decision head.
|
| 9 |
+
|
| 10 |
+
Read [the training guide](references/training.md) for the data contract and exact commands. Work from a pinned checkout of `https://github.com/murabcd/fev` and a complete, verified `murabcd/fev-4b` package. The checkout must contain `fev4b/train.py`; older releases expose inference only.
|
| 11 |
+
|
| 12 |
+
Before allocating compute:
|
| 13 |
+
|
| 14 |
+
1. Identify a reproducible weakness on examples excluded from training. Changing a customer's categories or rules normally requires runtime configuration, not training.
|
| 15 |
+
2. Agree on one bounded experiment, hardware/time cap and a pass/fail rule. Check current provider pricing and pending jobs; persist allocation intent and job ID. Reconcile an uncertain submission before retrying.
|
| 16 |
+
3. Use shared data spanning multiple domains. Group paraphrases, source documents and generated families together; keep train, development, calibration and test separate. The splitter catches exact-state/group overlap, not semantic duplicates. Inspect synthetic labels and describe their limitations.
|
| 17 |
+
4. Run the released baseline with the same questions, precision and routing policy. Keep test data out of training, tuning and calibration. Never send customer data to an external service merely because this skill is installed.
|
| 18 |
+
|
| 19 |
+
Training writes an explicit overlay directory; keep the downloaded base immutable. Verify finite gradients, adapter changes, frozen backbone/readout, save/reload agreement and complete predictions. Fit temperature on calibration data only. Inspect accuracy, Brier/NLL, confident mistakes and coverage together. Temperature can change threshold-based routing even when option argmax stays the same.
|
| 20 |
+
|
| 21 |
+
Use an untouched cross-domain test for adoption. A toy training smoke check establishes mechanics, not generalization or improved accuracy. Do not automatically publish, deploy, modify customer workflows or replace released weights. Report the evidence and obtain the user's requested release decision.
|
skills/fev-finetune/references/training.md
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Training Fev-4B
|
| 2 |
+
|
| 3 |
+
Training is optional. Customer categories, business rules, examples and actual conversation history are runtime inputs. The default Fev checkpoint remains shared across customers. This workflow continues the released rank-8, alpha-16 adapter (10,387,456 FP32 parameters); the BF16 backbone and native readout stay frozen. It uses decision cross-entropy, not next-token/chat SFT.
|
| 4 |
+
|
| 5 |
+
## Environment and baseline
|
| 6 |
+
|
| 7 |
+
Use Linux, Python 3.12 and a BF16-capable NVIDIA GPU. An A10G 24 GB is the research reference. The short smoke workload does not establish memory requirements at the 16,384-token limit. An 8 GB Mac can prepare data and run contract tests; use a GPU server or a bounded Hugging Face Job for training.
|
| 8 |
+
|
| 9 |
+
In a checkout containing the training tools:
|
| 10 |
+
|
| 11 |
+
```bash
|
| 12 |
+
uv venv --python 3.12
|
| 13 |
+
uv pip install -r requirements.txt -r requirements-test.txt
|
| 14 |
+
uvx --from huggingface-hub==1.31.0 hf download murabcd/fev-4b \
|
| 15 |
+
--revision v0.2.0 --local-dir models/fev-4b
|
| 16 |
+
.venv/bin/python run.py --model-dir models/fev-4b --verify-only
|
| 17 |
+
```
|
| 18 |
+
|
| 19 |
+
The unchanged v0.2.0 package is an explicit baseline. Pin its immutable commit for a recorded experiment. Do not train on published benchmark inputs. Do not interpret multiple-domain metadata alone as evidence of generalization.
|
| 20 |
+
|
| 21 |
+
## Labelled data
|
| 22 |
+
|
| 23 |
+
Use one JSON object per line. `id` must be unique. `domain` identifies the application domain; `group` joins a document, paraphrase family or generation template so it cannot cross splits. Each question includes its normal typed request and a `label`:
|
| 24 |
+
|
| 25 |
+
```json
|
| 26 |
+
{"id":"library-001","group":"library-reservation-family-a","domain":"library","state":{"request":"Отмените бронь книги","reserved":2,"limit":4},"questions":{"action":{"type":"choice","instructions":"Выберите текущую просьбу.","criteria":{"cancel":"Отменить бронь","change":"Изменить бронь","info":"Запросить информацию"},"label":"cancel"},"capacity":{"type":"noul","instructions":"Количество reserved меньше limit?","label":true},"load":{"type":"score","instructions":"Оцените reserved.","criteria":["Меньше 2","От 2 до 4 включительно","Больше 4"],"label":1}}}
|
| 27 |
+
```
|
| 28 |
+
|
| 29 |
+
Choice labels are option keys; Noul labels are JSON booleans; Score labels are zero-based integer indices into the rubric. `label` and dataset metadata are removed before inference. Include `domain` on every training record and at least two training domains. Use group IDs across domains when one template/source produced both. Review ambiguous definitions and labels instead of letting a model's prediction become its own gold answer.
|
| 30 |
+
|
| 31 |
+
`examples/training.jsonl` contains controlled toy examples for mechanics only. It is not a production training corpus or a quality benchmark.
|
| 32 |
+
|
| 33 |
+
```bash
|
| 34 |
+
.venv/bin/python -m fev4b.data --data examples/training.jsonl --out work/split --seed 42
|
| 35 |
+
```
|
| 36 |
+
|
| 37 |
+
The deterministic split uses connected exact states and supplied groups: about 70% training and 10% each development, calibration and test by group count. It needs at least four independent groups. It does not detect fuzzy duplicates; choose meaningful family groups and audit the result. For cross-domain evidence, also hold out entire domains explicitly rather than relying on a random group split.
|
| 38 |
+
|
| 39 |
+
## Train, calibrate, test
|
| 40 |
+
|
| 41 |
+
Run a baseline on the frozen development partition, then take one predeclared bounded run. The trainer records its own development baseline and rejects overlap, oversized requests and a changed adapter architecture. It never truncates rules and never writes into the original model directory.
|
| 42 |
+
|
| 43 |
+
```bash
|
| 44 |
+
.venv/bin/python -m fev4b.train \
|
| 45 |
+
--model-dir models/fev-4b \
|
| 46 |
+
--train work/split/train.jsonl --development work/split/development.jsonl \
|
| 47 |
+
--steps 6 --learning-rate 0.00002 --seed 42 --out work/adapter-smoke
|
| 48 |
+
|
| 49 |
+
.venv/bin/python -m fev4b.calibrate \
|
| 50 |
+
--model-dir models/fev-4b --adaptation-dir work/adapter-smoke \
|
| 51 |
+
--data work/split/calibration.jsonl --out work/calibrated-smoke
|
| 52 |
+
|
| 53 |
+
.venv/bin/python -m fev4b.benchmark \
|
| 54 |
+
--model-dir models/fev-4b --adaptation-dir work/calibrated-smoke \
|
| 55 |
+
--data work/split/test.jsonl --out work/test-once
|
| 56 |
+
```
|
| 57 |
+
|
| 58 |
+
Six steps on the toy file test the machinery; they are not a recommended training duration or an improved released model. Select step count and learning rate using development data before opening a final test. Output directories must be new. Preserve failed-run evidence instead of overwriting it.
|
| 59 |
+
|
| 60 |
+
The trainer verifies adapter-only gradients, frozen parameter versions, finite losses and an actual adapter update. Its outputs include weights/config, `adaptation.json`, baseline/adapted predictions and a training report. The loader checks the base identity, adapter configuration, tensor keys/shapes/dtypes, finite values and payload SHA256. Checksums detect accidental changes; they do not authenticate an untrusted publisher.
|
| 61 |
+
|
| 62 |
+
Calibration fits one positive temperature by calibration NLL. It preserves per-question option ranking mathematically, but can change routing cutoffs, coverage and applicability decisions. Evaluate the complete routing composition before adopting a calibrated overlay. The base loader's fixed thresholds are not correctness guarantees.
|
| 63 |
+
|
| 64 |
+
The benchmark reports all questions in the accuracy denominator, including failures; Brier/NLL/ECE use answered questions and disclose their count. It also reports fixed-threshold coverage and errors. For Score, probabilities and expected ordinal position matter as well as top-level accuracy. ECE on a small or synthetic sample is not proof of general calibration.
|
| 65 |
+
|
| 66 |
+
Compare unchanged baseline and candidate on exactly the same untouched test data. For a predeclared final comparison, also save the unchanged baseline:
|
| 67 |
+
|
| 68 |
+
```bash
|
| 69 |
+
.venv/bin/python -m fev4b.benchmark --model-dir models/fev-4b \
|
| 70 |
+
--data work/split/test.jsonl --out work/baseline-test
|
| 71 |
+
```
|
| 72 |
+
|
| 73 |
+
The baseline and candidate may both read the frozen test in that final comparison; do not use the result to tune and retest the same candidate family. Retain original labels and raw responses, inspect confident mistakes, and report denominators separately by source/type/domain. Do not select among repeated candidates on the final test. A failed generalization check is a reason to retain the baseline.
|
| 74 |
+
|
| 75 |
+
## Load an explicit overlay
|
| 76 |
+
|
| 77 |
+
```bash
|
| 78 |
+
.venv/bin/python serve.py --model-dir models/fev-4b \
|
| 79 |
+
--adaptation-dir work/calibrated-smoke --port 8000
|
| 80 |
+
```
|
| 81 |
+
|
| 82 |
+
No overlay is discovered or adopted automatically. It must match the exact packaged checkpoint/original adapter. Do not distribute mechanics-only toy overlays as a new Fev release.
|
| 83 |
+
|
| 84 |
+
## Hugging Face GPU Job
|
| 85 |
+
|
| 86 |
+
The repository provides `tools/train_hf.py`, a UV script with pinned dependencies. It downloads an immutable model revision and uses a separate source ZIP and four input partitions. Outputs go only to the mounted output directory. It does not upload to a model repository.
|
| 87 |
+
|
| 88 |
+
Prepare a source ZIP containing `fev/` and `fev4b/` Python files from the same reviewed checkout; exclude credentials, environments and weights. Put it at `cloud-inputs/source.zip` beside the four JSONL files. Record its SHA256 and the data hashes before allocation. Run `hf auth whoami`, inspect `hf jobs hardware` pricing, and create/use a **private** output bucket. Keep a timeout and record the returned job ID:
|
| 89 |
+
|
| 90 |
+
```bash
|
| 91 |
+
hf jobs uv run --flavor a10g-small --timeout 20m --detach \
|
| 92 |
+
--volume ./cloud-inputs:/inputs \
|
| 93 |
+
--volume hf://buckets/YOUR_USER/YOUR_PRIVATE_BUCKET:/outputs:rw \
|
| 94 |
+
tools/train_hf.py --inputs /inputs --output /outputs/UNIQUE_RUN \
|
| 95 |
+
--revision 18ab9ad9f0000345eade9f001f16085ef4a34370 --steps 6
|
| 96 |
+
```
|
| 97 |
+
|
| 98 |
+
Local-directory mounts upload their contents to HF. Use only data approved for that destination. The example performs one training/calibration/test sequence; it does not run a tuning loop. The output path must be new. A job timeout limits runtime, not all account costs; check current pricing first. Inspect logs/results and confirm the job finished before allocating a follow-up. Production inference can remain wholly inside the required region after downloading files and dependencies.
|
tests/test_training_tools.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Leakage, label and measurement checks independent of CUDA."""
|
| 2 |
+
import copy
|
| 3 |
+
import json
|
| 4 |
+
import tempfile
|
| 5 |
+
import unittest
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
from fev4b.data import split_records, require_disjoint, target_index
|
| 9 |
+
from fev4b.metrics import rows_for, summarize
|
| 10 |
+
from fev4b.calibrate import fit_temperature, rescale
|
| 11 |
+
from fev4b.benchmark import evaluate
|
| 12 |
+
from fev4b.adaptation import base_identity, verify_adaptation
|
| 13 |
+
from fev4b.integrity import file_sha256
|
| 14 |
+
from tools.compare_runs import compare
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def record(i, group=None):
|
| 18 |
+
return {'id': str(i), 'group': group or str(i), 'state': str(i), 'questions': {
|
| 19 |
+
'q': {'type': 'choice', 'criteria': {'yes': None, 'no': None}, 'label': 'yes'}}}
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class TrainingToolsTests(unittest.TestCase):
|
| 23 |
+
def test_overlay_rejects_other_base_or_modified_payload(self):
|
| 24 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 25 |
+
root = Path(directory)
|
| 26 |
+
(root/'adapter').mkdir()
|
| 27 |
+
config = {'r': 8, 'lora_alpha': 16}
|
| 28 |
+
(root/'adapter/adapter_config.json').write_text(json.dumps(config))
|
| 29 |
+
manifest = {'files': {'checkpoint/model.safetensors': {'bytes': 4, 'sha256': 'base'},
|
| 30 |
+
'adapter/adapter_model.safetensors': {'bytes': 4, 'sha256': 'shared'}}}
|
| 31 |
+
(root/'manifest.json').write_text(json.dumps(manifest))
|
| 32 |
+
overlay = root/'overlay'
|
| 33 |
+
overlay.mkdir()
|
| 34 |
+
(overlay/'adapter_config.json').write_text(json.dumps(config))
|
| 35 |
+
(overlay/'adapter_model.safetensors').write_bytes(b'test')
|
| 36 |
+
meta = {'format': 'fev-adaptation-v1', 'baseIdentity': base_identity(root), 'temperature': 1.5,
|
| 37 |
+
'files': {name: file_sha256(overlay/name) for name in
|
| 38 |
+
('adapter_config.json', 'adapter_model.safetensors')}}
|
| 39 |
+
(overlay/'adaptation.json').write_text(json.dumps(meta))
|
| 40 |
+
self.assertEqual(verify_adaptation(root, overlay)['temperature'], 1.5)
|
| 41 |
+
(overlay/'adapter_model.safetensors').write_bytes(b'changed')
|
| 42 |
+
with self.assertRaises(ValueError):
|
| 43 |
+
verify_adaptation(root, overlay)
|
| 44 |
+
(overlay/'adapter_model.safetensors').write_bytes(b'test')
|
| 45 |
+
manifest['files']['checkpoint/model.safetensors']['sha256'] = 'other'
|
| 46 |
+
(root/'manifest.json').write_text(json.dumps(manifest))
|
| 47 |
+
with self.assertRaises(ValueError):
|
| 48 |
+
verify_adaptation(root, overlay)
|
| 49 |
+
|
| 50 |
+
def test_comparison_rejects_different_labels_and_preserves_paired_counts(self):
|
| 51 |
+
a = rows_for(record(0), {'answers': {'q': {'type': 'choice', 'probabilities': {'yes': .9, 'no': .1}}}})
|
| 52 |
+
b = rows_for(record(0), {'answers': {'q': {'type': 'choice', 'probabilities': {'yes': .1, 'no': .9}}}})
|
| 53 |
+
result = compare({'a': a, 'b': b})
|
| 54 |
+
self.assertEqual(result['pairedOutcomes']['a vs b']['aOnlyCorrect'], 1)
|
| 55 |
+
b[0]['target'] = 1
|
| 56 |
+
with self.assertRaises(ValueError):
|
| 57 |
+
compare({'a': a, 'b': b})
|
| 58 |
+
|
| 59 |
+
def test_group_bridge_does_not_leak(self):
|
| 60 |
+
records = [record(i) for i in range(30)]
|
| 61 |
+
duplicate = record(31, group='0')
|
| 62 |
+
duplicate['state'] = '1'
|
| 63 |
+
records.append(duplicate)
|
| 64 |
+
split = split_records(records)
|
| 65 |
+
require_disjoint(*split.values())
|
| 66 |
+
partition = next(rows for rows in split.values() if any(r['id'] == '0' for r in rows))
|
| 67 |
+
self.assertTrue({'0', '1', '31'} <= {r['id'] for r in partition})
|
| 68 |
+
with self.assertRaises(ValueError):
|
| 69 |
+
require_disjoint([record(0)], [record(1, group='0')])
|
| 70 |
+
|
| 71 |
+
def test_label_types_are_strict(self):
|
| 72 |
+
with self.assertRaises(ValueError):
|
| 73 |
+
target_index({'type': 'noul', 'label': 1})
|
| 74 |
+
with self.assertRaises(ValueError):
|
| 75 |
+
target_index({'type': 'score', 'criteria': ['low', 'high'], 'label': True})
|
| 76 |
+
|
| 77 |
+
def test_errors_remain_in_denominator_and_confidence_is_top_probability(self):
|
| 78 |
+
response = {'answers': {'q': {'type': 'choice', 'probabilities': {'yes': .9, 'no': .1}}}}
|
| 79 |
+
rows = rows_for(record(0), response) + rows_for(record(1), error='Timeout')
|
| 80 |
+
report = summarize(rows)
|
| 81 |
+
self.assertEqual(report['accuracyAllQuestions'], .5)
|
| 82 |
+
self.assertEqual(report['coverageByFixedThreshold'][3]['coverage'], .5)
|
| 83 |
+
response['answers']['q']['probabilities'] = {'yes': 2., 'no': -1.}
|
| 84 |
+
with self.assertRaises(ValueError):
|
| 85 |
+
rows_for(record(0), response)
|
| 86 |
+
|
| 87 |
+
def test_temperature_does_not_change_argmax(self):
|
| 88 |
+
values = [.05, .15, .8]
|
| 89 |
+
for factor in (.2, 1., 5.):
|
| 90 |
+
self.assertEqual(max(range(3), key=rescale(values, factor).__getitem__), 2)
|
| 91 |
+
rows = [{'error': None, 'probabilities': [.99, .01], 'target': 1} for _ in range(3)]
|
| 92 |
+
_, report = fit_temperature(rows)
|
| 93 |
+
self.assertLessEqual(report['afterNll'], report['beforeNll'])
|
| 94 |
+
|
| 95 |
+
def test_benchmark_keeps_failed_records(self):
|
| 96 |
+
def predict(request):
|
| 97 |
+
if request['state'] == '1':
|
| 98 |
+
raise RuntimeError('failed')
|
| 99 |
+
return {'model': 'fake', 'answers': {'q': {'type': 'choice', 'probabilities': {'yes': .8, 'no': .2}}}}
|
| 100 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 101 |
+
report = evaluate([record(0), record(1)], predict, Path(directory)/'run', provenance={})
|
| 102 |
+
self.assertEqual(report['overall']['questions'], 2)
|
| 103 |
+
self.assertEqual(report['overall']['errors'], 1)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
if __name__ == '__main__':
|
| 107 |
+
unittest.main()
|
tools/benchmark_kev.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# /// script
|
| 2 |
+
# requires-python = ">=3.12,<3.13"
|
| 3 |
+
# dependencies = [
|
| 4 |
+
# "kev @ git+https://github.com/jaredpalmer/kev.git@2074cc44256200bb37540103fa4572901faa10f3",
|
| 5 |
+
# "torch==2.8.0", "transformers==5.17.0", "peft==0.21.2",
|
| 6 |
+
# "accelerate==1.15.0", "numpy==2.5.3", "datasets==5.0.1",
|
| 7 |
+
# "scikit-learn==1.9.1", "pydantic==2.13.5", "huggingface-hub==1.31.0",
|
| 8 |
+
# "safetensors==0.8.0", "flash-linear-attention==0.5.2", "einops==0.8.1",
|
| 9 |
+
# ]
|
| 10 |
+
# ///
|
| 11 |
+
"""Run the pinned official Kev BF16 reference path on Fev's common JSONL."""
|
| 12 |
+
import argparse
|
| 13 |
+
import hashlib
|
| 14 |
+
import sys
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def main():
|
| 19 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 20 |
+
parser.add_argument('--data', type=Path, required=True)
|
| 21 |
+
parser.add_argument('--out', type=Path, required=True)
|
| 22 |
+
args = parser.parse_args()
|
| 23 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
| 24 |
+
import torch
|
| 25 |
+
if not torch.cuda.is_available():
|
| 26 |
+
raise RuntimeError('This measured path needs a CUDA GPU; no CPU/MPS comparison is qualified')
|
| 27 |
+
from huggingface_hub import snapshot_download
|
| 28 |
+
from kev.api import SystemOneRequest, to_record
|
| 29 |
+
from kev.checkpoint import Checkpoint, LoadOptions
|
| 30 |
+
from fev4b.data import load_records
|
| 31 |
+
from fev4b.benchmark import evaluate
|
| 32 |
+
revision = '6cfce5c2fa4b4bd64026336ab649c5ca78857d52'
|
| 33 |
+
torch.set_num_threads(4)
|
| 34 |
+
path = snapshot_download('jaredpalmer/kev-4b', revision=revision)
|
| 35 |
+
checkpoint = Checkpoint(path)
|
| 36 |
+
tok, model = checkpoint.load('cuda', LoadOptions(dtype=torch.bfloat16, merge=True,
|
| 37 |
+
attn='sdpa', backend='torch', fused=False, cuda_graphs=False))
|
| 38 |
+
def predict(request):
|
| 39 |
+
rec, meta = to_record(SystemOneRequest(**request))
|
| 40 |
+
enc = model.encode(tok, rec, max_state=16384, max_branch=16384, strict=True)
|
| 41 |
+
if len(enc['ids']) > 16384:
|
| 42 |
+
raise ValueError('Comparison single-request token limit exceeded')
|
| 43 |
+
with torch.inference_mode():
|
| 44 |
+
logits = model.forward(enc)
|
| 45 |
+
answers = {}
|
| 46 |
+
for key, q, z in zip(request['questions'], meta, logits, strict=True):
|
| 47 |
+
probabilities = z.float().softmax(-1).cpu().tolist()
|
| 48 |
+
if q['type'] == 'noul':
|
| 49 |
+
answers[key] = {'type': 'noul', 'noul': probabilities[1]}
|
| 50 |
+
else:
|
| 51 |
+
answers[key] = {'type': q['type'], 'probabilities': dict(zip(q['keys'], probabilities, strict=True))}
|
| 52 |
+
return {'model': 'kev-4b@'+revision, 'answers': answers}
|
| 53 |
+
report = evaluate(load_records(args.data), predict, args.out, provenance={
|
| 54 |
+
'revision': revision, 'codeRevision': '2074cc44256200bb37540103fa4572901faa10f3',
|
| 55 |
+
'dataSha256': hashlib.sha256(args.data.read_bytes()).hexdigest(),
|
| 56 |
+
'precision': 'BF16 merged, official eager SDPA; no graphs/fused',
|
| 57 |
+
'gpu': torch.cuda.get_device_name(), 'torch': torch.__version__, 'temperature': model.head.temperature})
|
| 58 |
+
if report['overall']['errors']:
|
| 59 |
+
raise RuntimeError('Comparison has failed questions; inspect saved results')
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
if __name__ == '__main__':
|
| 63 |
+
main()
|
tools/compare_runs.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Summarize saved native typed runs without requerying or tuning a model."""
|
| 2 |
+
import argparse
|
| 3 |
+
import json
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
from fev4b.metrics import summarize
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def compare(runs):
|
| 10 |
+
indexed = {}
|
| 11 |
+
for name, rows in runs.items():
|
| 12 |
+
values = {(r['id'], r['question']): r for r in rows}
|
| 13 |
+
if len(values) != len(rows):
|
| 14 |
+
raise ValueError('Duplicate question in saved run')
|
| 15 |
+
indexed[name] = values
|
| 16 |
+
keys = next(iter(indexed.values())).keys()
|
| 17 |
+
if any(values.keys() != keys for values in indexed.values()):
|
| 18 |
+
raise ValueError('Runs do not contain the same questions')
|
| 19 |
+
for key in keys:
|
| 20 |
+
labels = {(v[key]['target'], v[key]['type'], v[key]['source'])
|
| 21 |
+
for v in indexed.values() if v[key]['error'] is None}
|
| 22 |
+
if len(labels) > 1:
|
| 23 |
+
raise ValueError('Runs disagree on target/type/source metadata')
|
| 24 |
+
results = {}
|
| 25 |
+
for name, rows in runs.items():
|
| 26 |
+
subsets = {'all': rows, 'publicTransfer': [r for r in rows if r['source'] != 'authored-russian-typed'],
|
| 27 |
+
'authoredRussian': [r for r in rows if r['source'] == 'authored-russian-typed']}
|
| 28 |
+
results[name] = {}
|
| 29 |
+
for label, values in subsets.items():
|
| 30 |
+
summary = summarize(values)
|
| 31 |
+
ordinal = [r['ordinalAbsoluteError'] for r in values
|
| 32 |
+
if r['error'] is None and 'ordinalAbsoluteError' in r]
|
| 33 |
+
summary['ordinalMeanAbsoluteError'] = sum(ordinal)/len(ordinal) if ordinal else None
|
| 34 |
+
results[name][label] = summary
|
| 35 |
+
results[name]['byType'] = {t:summarize([r for r in rows if r['type']==t]) for t in sorted({r['type'] for r in rows})}
|
| 36 |
+
results[name]['bySource'] = {s:summarize([r for r in rows if r['source']==s]) for s in sorted({r['source'] for r in rows})}
|
| 37 |
+
paired = {}
|
| 38 |
+
names = list(runs)
|
| 39 |
+
for i, a in enumerate(names):
|
| 40 |
+
for b in names[i+1:]:
|
| 41 |
+
av, bv = indexed[a], indexed[b]
|
| 42 |
+
paired[a+' vs '+b] = {
|
| 43 |
+
'aOnlyCorrect': sum(bool(av[k].get('correct')) and not bv[k].get('correct') for k in keys),
|
| 44 |
+
'bOnlyCorrect': sum(bool(bv[k].get('correct')) and not av[k].get('correct') for k in keys),
|
| 45 |
+
'bothCorrect': sum(bool(av[k].get('correct')) and bool(bv[k].get('correct')) for k in keys),
|
| 46 |
+
'neitherCorrect': sum(not av[k].get('correct') and not bv[k].get('correct') for k in keys)}
|
| 47 |
+
return {'models': results, 'pairedOutcomes': paired,
|
| 48 |
+
'interpretation': 'Descriptive comparison on fixed public development and repetitive authored inputs; no significance or unseen customer accuracy claim'}
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def main():
|
| 52 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 53 |
+
parser.add_argument('--run', action='append', required=True, help='NAME=run-directory; repeat for each system')
|
| 54 |
+
parser.add_argument('--out', type=Path, required=True)
|
| 55 |
+
args = parser.parse_args()
|
| 56 |
+
if args.out.exists():
|
| 57 |
+
raise ValueError('Output already exists')
|
| 58 |
+
runs = {}
|
| 59 |
+
provenance = {}
|
| 60 |
+
hashes = set()
|
| 61 |
+
for spec in args.run:
|
| 62 |
+
name, directory = spec.split('=', 1)
|
| 63 |
+
if name in runs:
|
| 64 |
+
raise ValueError('Duplicate system name')
|
| 65 |
+
directory = Path(directory)
|
| 66 |
+
report = json.loads((directory/'report.json').read_text())
|
| 67 |
+
p = report['provenance']
|
| 68 |
+
digest = p.get('dataSha256', p.get('datasetSha256'))
|
| 69 |
+
if not digest:
|
| 70 |
+
raise ValueError('Missing input data provenance')
|
| 71 |
+
hashes.add(digest)
|
| 72 |
+
runs[name] = json.loads((directory/'rows.json').read_text())
|
| 73 |
+
provenance[name] = p
|
| 74 |
+
if len(hashes) != 1:
|
| 75 |
+
raise ValueError('Runs use different datasets')
|
| 76 |
+
result = {**compare(runs), 'datasetSha256': hashes.pop(), 'provenance': provenance}
|
| 77 |
+
args.out.parent.mkdir(parents=True, exist_ok=True)
|
| 78 |
+
args.out.write_text(json.dumps(result, indent=2)+'\n')
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
if __name__ == '__main__':
|
| 82 |
+
main()
|
tools/prepare_benchmark.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Fetch the pinned public development suite and append authored Russian probes."""
|
| 2 |
+
import argparse
|
| 3 |
+
import hashlib
|
| 4 |
+
import json
|
| 5 |
+
import urllib.request
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
REVISION = '2074cc44256200bb37540103fa4572901faa10f3'
|
| 9 |
+
ORIGINAL_SHA256 = 'ff374c49c6c9f15f8a56fb274b4a4857d20497eb8dd1ac07ce01560e682a5f2e'
|
| 10 |
+
COMPARISON_SHA256 = '9266d6ea1ce83c9755a60cf48c7469b0fa3264c229e96f4142d683bada8fef91'
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def main():
|
| 14 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 15 |
+
parser.add_argument('--out', type=Path, required=True)
|
| 16 |
+
args = parser.parse_args()
|
| 17 |
+
if args.out.exists():
|
| 18 |
+
raise ValueError('Output must be a new directory')
|
| 19 |
+
url = f'https://raw.githubusercontent.com/jaredpalmer/kev/{REVISION}/evals/v4/transfer-v4/development.jsonl'
|
| 20 |
+
with urllib.request.urlopen(url, timeout=60) as response:
|
| 21 |
+
data = response.read()
|
| 22 |
+
if hashlib.sha256(data).hexdigest() != ORIGINAL_SHA256:
|
| 23 |
+
raise ValueError('Pinned upstream dataset hash mismatch')
|
| 24 |
+
converted = []
|
| 25 |
+
for i, line in enumerate(data.decode().splitlines()):
|
| 26 |
+
row = json.loads(line)
|
| 27 |
+
converted.append({'id': f'transfer-v4-{i:04}', 'source': row['_meta']['source'],
|
| 28 |
+
'state': row['state'], 'questions': {k: {n:v for n,v in q.items() if n != 'src'}
|
| 29 |
+
for k,q in row['questions'].items()}})
|
| 30 |
+
text = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in converted)
|
| 31 |
+
russian = Path(__file__).resolve().parents[1]/'evaluation/benchmarks/russian-typed.jsonl'
|
| 32 |
+
combined = (text+russian.read_text()).encode()
|
| 33 |
+
if len(converted) != 764 or hashlib.sha256(combined).hexdigest() != COMPARISON_SHA256:
|
| 34 |
+
raise ValueError('Converted comparison hash mismatch')
|
| 35 |
+
args.out.mkdir(parents=True)
|
| 36 |
+
(args.out/'transfer-v4.jsonl').write_text(text)
|
| 37 |
+
(args.out/'comparison.jsonl').write_bytes(combined)
|
| 38 |
+
(args.out/'manifest.json').write_text(json.dumps({'url':url, 'upstreamSha256':ORIGINAL_SHA256,
|
| 39 |
+
'comparisonSha256':COMPARISON_SHA256, 'records':812, 'questions':908,
|
| 40 |
+
'usage':'Development comparison, not training or fresh customer gold',
|
| 41 |
+
'rights':'Upstream data retains its original source licenses; review Kev transfer-v4 source manifest'}, indent=2)+'\n')
|
| 42 |
+
print(COMPARISON_SHA256)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
if __name__ == '__main__':
|
| 46 |
+
main()
|
tools/train_hf.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# /// script
|
| 2 |
+
# requires-python = ">=3.12,<3.13"
|
| 3 |
+
# dependencies = [
|
| 4 |
+
# "torch==2.14.0", "torchvision==0.29.0", "transformers==5.17.0",
|
| 5 |
+
# "accelerate==1.10.1", "safetensors==0.8.0", "huggingface-hub==1.31.0",
|
| 6 |
+
# "pillow==12.3.0", "flash-linear-attention==0.5.2", "einops==0.8.1", "peft==0.21.2",
|
| 7 |
+
# ]
|
| 8 |
+
# ///
|
| 9 |
+
"""One shared-adapter experiment; private mounted artifacts, no model publishing."""
|
| 10 |
+
import argparse
|
| 11 |
+
import hashlib
|
| 12 |
+
import json
|
| 13 |
+
import subprocess
|
| 14 |
+
import sys
|
| 15 |
+
import zipfile
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def main():
|
| 20 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 21 |
+
parser.add_argument('--inputs', type=Path, required=True)
|
| 22 |
+
parser.add_argument('--output', type=Path, required=True)
|
| 23 |
+
parser.add_argument('--revision', required=True)
|
| 24 |
+
parser.add_argument('--repo', default='murabcd/fev-4b')
|
| 25 |
+
parser.add_argument('--steps', type=int, default=6)
|
| 26 |
+
args = parser.parse_args()
|
| 27 |
+
if len(args.revision) != 40 or any(c not in '0123456789abcdef' for c in args.revision):
|
| 28 |
+
raise ValueError('Pin an immutable 40-character Hub commit')
|
| 29 |
+
args.output.mkdir(parents=True, exist_ok=False)
|
| 30 |
+
names = ('source.zip', 'train.jsonl', 'development.jsonl', 'calibration.jsonl', 'test.jsonl')
|
| 31 |
+
intent = {'repo': args.repo, 'revision': args.revision, 'steps': args.steps,
|
| 32 |
+
'inputHashes': {n: hashlib.sha256((args.inputs/n).read_bytes()).hexdigest() for n in names},
|
| 33 |
+
'published': False, 'automaticAdoption': False}
|
| 34 |
+
(args.output/'intent.json').write_text(json.dumps(intent, indent=2)+'\n')
|
| 35 |
+
code = Path('/tmp/fev-training-source')
|
| 36 |
+
with zipfile.ZipFile(args.inputs/'source.zip') as archive:
|
| 37 |
+
if any(Path(n).is_absolute() or '..' in Path(n).parts or
|
| 38 |
+
not n.startswith(('fev/', 'fev4b/')) or not n.endswith('.py') for n in archive.namelist()):
|
| 39 |
+
raise ValueError('Source ZIP must contain only fev/ and fev4b/ Python files')
|
| 40 |
+
archive.extractall(code)
|
| 41 |
+
from huggingface_hub import snapshot_download
|
| 42 |
+
model = snapshot_download(args.repo, revision=args.revision, local_dir='/tmp/fev-training-base')
|
| 43 |
+
def run(module, *parameters):
|
| 44 |
+
subprocess.run([sys.executable, '-m', module, '--model-dir', model, *map(str, parameters)],
|
| 45 |
+
cwd=code, check=True)
|
| 46 |
+
run('fev4b.train', '--train', args.inputs/'train.jsonl', '--development', args.inputs/'development.jsonl',
|
| 47 |
+
'--steps', args.steps, '--out', args.output/'adapter')
|
| 48 |
+
run('fev4b.calibrate', '--adaptation-dir', args.output/'adapter', '--data', args.inputs/'calibration.jsonl',
|
| 49 |
+
'--out', args.output/'calibrated')
|
| 50 |
+
run('fev4b.benchmark', '--adaptation-dir', args.output/'calibrated', '--data', args.inputs/'test.jsonl',
|
| 51 |
+
'--out', args.output/'test')
|
| 52 |
+
report = json.loads((args.output/'test/report.json').read_text())
|
| 53 |
+
if report['overall']['errors']:
|
| 54 |
+
raise RuntimeError('Incomplete test predictions; do not adopt')
|
| 55 |
+
(args.output/'complete.json').write_text(json.dumps({'complete': True, 'adopted': False})+'\n')
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
if __name__ == '__main__':
|
| 59 |
+
main()
|