MyeongHoJeong commited on
Commit
978f974
·
verified ·
1 Parent(s): a6bc6dd

v3 card: correct the long-document and JevBench hard numbers (clean re-run of the released weights)

Browse files
Files changed (3) hide show
  1. README.md +2 -2
  2. SHA256SUMS +2 -2
  3. release-manifest.json +7 -7
README.md CHANGED
@@ -56,8 +56,8 @@ graph, and returns a calibrated distribution for each question at once. In pract
56
  | Areas: Knowledge / Language / Retrieval / Tools / Arts | 25.2 / 44.2 / 59.7 / 60.3 / 19.8 | 30.3 / 49.5 / 63.3 / 79.8 / 36.5 |
57
  | HoVer, multi-hop claim verification (DI) | 62.7 | 63.9 |
58
  | BRIGHT, reasoning-heavy retrieval over long documents, ~12k tokens per request (DI) | 25.7 | 23.9 |
59
- | Decision hidden in a long document (our set, 150 items) | 82.00 | 88.00 |
60
- | JevBench public: easy / original / hard (48 / 72 / 111 items) | 100.00 / 98.61 / 63.96 | 100.00 / 98.61 / 60.36 |
61
 
62
  Decision Index rows (DI) are chance-corrected skill scores (0 = random guessing, 100 = perfect) from our own runs of the public
63
  0.3 suite with the official kit, all 140,178 scoreable requests; every benchmark of both runs is in [standardone-3b-sh-v2](https://huggingface.co/datasets/StandardThinking/StandardOne-SH-decision-index/tree/main/runs/standardone-3b-sh-v2)
 
56
  | Areas: Knowledge / Language / Retrieval / Tools / Arts | 25.2 / 44.2 / 59.7 / 60.3 / 19.8 | 30.3 / 49.5 / 63.3 / 79.8 / 36.5 |
57
  | HoVer, multi-hop claim verification (DI) | 62.7 | 63.9 |
58
  | BRIGHT, reasoning-heavy retrieval over long documents, ~12k tokens per request (DI) | 25.7 | 23.9 |
59
+ | Decision hidden in a long document (our set, 150 items) | 82.00 | 85.33 |
60
+ | JevBench public: easy / original / hard (48 / 72 / 111 items) | 100.00 / 98.61 / 63.96 | 100.00 / 98.61 / 63.96 |
61
 
62
  Decision Index rows (DI) are chance-corrected skill scores (0 = random guessing, 100 = perfect) from our own runs of the public
63
  0.3 suite with the official kit, all 140,178 scoreable requests; every benchmark of both runs is in [standardone-3b-sh-v2](https://huggingface.co/datasets/StandardThinking/StandardOne-SH-decision-index/tree/main/runs/standardone-3b-sh-v2)
SHA256SUMS CHANGED
@@ -1,5 +1,5 @@
1
  1495e1e757ef4d0925a2350563cf5754bb23c51701a8ec4fb3c5cdcbedae6747 LICENSE
2
- e9b0729b5f196d2dd66ff424a18e02d7d8b12db6235ccb838d004416a40b069e README.md
3
  331b249682cd52226c50e533f59825184997f3b31f9a62b2c3fea940db6999c5 SYSTEM_PROMPT.txt
4
  e62f05da25555a4c109d0f413eb673d328fcc40f7c82537c61631611f6954372 chat_template.jinja
5
  3208d5b86b4ce169d39302f52944f8898e42c6917aab121524bd54cae5af9cb0 config.json
@@ -9,7 +9,7 @@ e0923390059f84a9180b00e5501778acc45ea9856cd7f2fd68208b360927c677 generation_con
9
  55641440eb385d8bc83a319b0a92f60fe45e6aef4b6992245eee91142237786e model.safetensors.index.json
10
  d15587c57574d2a92c66ff2552c6db5173d4c440ad47f7454dc51bad0ecaeb59 params.json
11
  ece2373c2ae391bce18785a1810543bc6173a1f28b6767cffab2c35dbea5f002 processor_config.json
12
- 535a89c61b8551783e056c09f5111df116823b2ea107ca18d260d64e78e997ee release-manifest.json
13
  0a5c981e8c5c6f8886ee007a6d4543a0be6b221cb9ca32a8709384a4c6fc8cbb special_tokens_map.json
14
  600bb27946565481ecf51ba8aee252e49b9a68507866080ac9c30185bb312843 tekken.json
15
  d5f6046775b112f0e2d456ee9dba450684ab964fe5c4e231599bdc6773028135 tokenizer.json
 
1
  1495e1e757ef4d0925a2350563cf5754bb23c51701a8ec4fb3c5cdcbedae6747 LICENSE
2
+ 320de6b44809039797794661b79d1ffd7ad7d96447f2fa3184af8f4010521843 README.md
3
  331b249682cd52226c50e533f59825184997f3b31f9a62b2c3fea940db6999c5 SYSTEM_PROMPT.txt
4
  e62f05da25555a4c109d0f413eb673d328fcc40f7c82537c61631611f6954372 chat_template.jinja
5
  3208d5b86b4ce169d39302f52944f8898e42c6917aab121524bd54cae5af9cb0 config.json
 
9
  55641440eb385d8bc83a319b0a92f60fe45e6aef4b6992245eee91142237786e model.safetensors.index.json
10
  d15587c57574d2a92c66ff2552c6db5173d4c440ad47f7454dc51bad0ecaeb59 params.json
11
  ece2373c2ae391bce18785a1810543bc6173a1f28b6767cffab2c35dbea5f002 processor_config.json
12
+ a48416ec35f1e50f5add103b1902c6dbb8b1380aa9ee73ace4b9173ed89ea30f release-manifest.json
13
  0a5c981e8c5c6f8886ee007a6d4543a0be6b221cb9ca32a8709384a4c6fc8cbb special_tokens_map.json
14
  600bb27946565481ecf51ba8aee252e49b9a68507866080ac9c30185bb312843 tekken.json
15
  d5f6046775b112f0e2d456ee9dba450684ab964fe5c4e231599bdc6773028135 tokenizer.json
release-manifest.json CHANGED
@@ -34,17 +34,17 @@
34
  "prompt_format": "chat template with an empty default system message (SH_SYSTEM_PROMPT=none)",
35
  "release": "StandardOne-3B-SH",
36
  "evaluation": {
37
- "engine": "Standard One engine (SGLang fork ad26e8b179), 1 x AMD Instinct MI350X, 2026-10-13",
38
  "engine_clean_accuracy": {
39
- "internal-eval-v1.heldout": 84.0,
40
- "internal-eval-hardproxy-v1.heldout": 38.83,
41
  "internal-eval-v2-realistic.heldout": 89.17,
42
  "jevbench-easy": 100.0,
43
  "jevbench-original": 98.61,
44
- "jevbench-hard": 60.36,
45
- "longctx-dev": 88.0,
46
- "winogrande-dev": 91.19,
47
- "repair5k-validation-policy-judgment": 56.5
48
  },
49
  "decision_index_0.3_public_index": 51.53,
50
  "decision_index_note": "self-scored with the official kit (apolinario/decision-index), all 140,178 scoreable requests"
 
34
  "prompt_format": "chat template with an empty default system message (SH_SYSTEM_PROMPT=none)",
35
  "release": "StandardOne-3B-SH",
36
  "evaluation": {
37
+ "engine": "Standard One engine (SGLang fork ad26e8b179), 1 x AMD Instinct MI350X, 2026-10-13, released weights (agrees with production on 1,427 of 1,431 parity questions)",
38
  "engine_clean_accuracy": {
39
+ "internal-eval-v1.heldout": 85.33,
40
+ "internal-eval-hardproxy-v1.heldout": 38.33,
41
  "internal-eval-v2-realistic.heldout": 89.17,
42
  "jevbench-easy": 100.0,
43
  "jevbench-original": 98.61,
44
+ "jevbench-hard": 63.96,
45
+ "longctx-dev": 85.33,
46
+ "winogrande-dev": 91.59,
47
+ "repair5k-validation-policy-judgment": 57.0
48
  },
49
  "decision_index_0.3_public_index": 51.53,
50
  "decision_index_note": "self-scored with the official kit (apolinario/decision-index), all 140,178 scoreable requests"