| { |
| "internal": { |
| "eval: [2] gen:all_conditions": { |
| "accuracy": 0.824, |
| "n": 34 |
| }, |
| "eval: [2] gen:dates": { |
| "accuracy": 0.871, |
| "n": 31 |
| }, |
| "eval: [2] gen:exception": { |
| "accuracy": 0.946, |
| "n": 37 |
| }, |
| "eval: [2] gen:lookup_chain": { |
| "accuracy": 0.8, |
| "n": 40 |
| }, |
| "eval: [2] gen:numbers": { |
| "accuracy": 0.848, |
| "n": 33 |
| }, |
| "eval: [2] gen:precedence": { |
| "accuracy": 0.917, |
| "n": 36 |
| }, |
| "eval: [2] gen:rubric": { |
| "accuracy": 0.794, |
| "n": 34 |
| }, |
| "eval: [2] gen:surface_trap": { |
| "accuracy": 0.958, |
| "n": 48 |
| }, |
| "eval: [2] gen:tradeoff": { |
| "accuracy": 0.7, |
| "n": 40 |
| }, |
| "eval: [2] gen:underdetermined": { |
| "accuracy": 0.765, |
| "n": 17 |
| }, |
| "eval: [3] gen:all_conditions": { |
| "accuracy": 0.889, |
| "n": 9 |
| }, |
| "eval: [3] gen:borderline_rubric": { |
| "accuracy": 0.538, |
| "n": 13 |
| }, |
| "eval: [3] gen:cross_artefact": { |
| "accuracy": 0.677, |
| "n": 31 |
| }, |
| "eval: [3] gen:dates": { |
| "accuracy": 0.8, |
| "n": 10 |
| }, |
| "eval: [3] gen:definition_scope": { |
| "accuracy": 0.667, |
| "n": 12 |
| }, |
| "eval: [3] gen:exception": { |
| "accuracy": 1.0, |
| "n": 15 |
| }, |
| "eval: [3] gen:judge_action": { |
| "accuracy": 0.714, |
| "n": 7 |
| }, |
| "eval: [3] gen:lookup_chain": { |
| "accuracy": 0.7, |
| "n": 10 |
| }, |
| "eval: [3] gen:missing_fact": { |
| "accuracy": 0.621, |
| "n": 29 |
| }, |
| "eval: [3] gen:nested_exception": { |
| "accuracy": 0.692, |
| "n": 13 |
| }, |
| "eval: [3] gen:numbers": { |
| "accuracy": 1.0, |
| "n": 1 |
| }, |
| "eval: [3] gen:precedence": { |
| "accuracy": 0.909, |
| "n": 11 |
| }, |
| "eval: [3] gen:rubric": { |
| "accuracy": 0.667, |
| "n": 3 |
| }, |
| "eval: [3] gen:running_total": { |
| "accuracy": 0.826, |
| "n": 23 |
| }, |
| "eval: [3] gen:surface_trap": { |
| "accuracy": 0.0, |
| "n": 1 |
| }, |
| "eval: [3] gen:tie_break": { |
| "accuracy": 0.826, |
| "n": 23 |
| }, |
| "eval: [3] gen:underdetermined": { |
| "accuracy": 1.0, |
| "n": 1 |
| }, |
| "eval: [3] gen:version_in_force": { |
| "accuracy": 0.704, |
| "n": 27 |
| }, |
| "eval: [3] gen:withdrawn_instruction": { |
| "accuracy": 0.75, |
| "n": 8 |
| }, |
| "eval: [4] adequacy_hs2": { |
| "accuracy": 0.739, |
| "n": 234 |
| }, |
| "eval: [5] gen:adequacy": { |
| "accuracy": 0.788, |
| "n": 302 |
| }, |
| "eval: boardgame": { |
| "accuracy": 0.821, |
| "n": 900 |
| }, |
| "eval: contractnli": { |
| "accuracy": 0.865, |
| "n": 1026 |
| }, |
| "eval: held-out short tasks": { |
| "accuracy": 0.65, |
| "ece": 0.06, |
| "n": 6000, |
| "nll": 0.851 |
| }, |
| "eval: hotpotqa (held out)": { |
| "accuracy": 0.746, |
| "n": 959 |
| }, |
| "eval: musique": { |
| "accuracy": 0.882, |
| "n": 1199 |
| }, |
| "eval: musique, answerable": { |
| "accuracy": 0.886, |
| "n": 599 |
| }, |
| "eval: musique, unanswerable": { |
| "accuracy": 0.877, |
| "n": 600 |
| } |
| }, |
| "jevbench": [ |
| { |
| "accuracy": 0.7619047619047619, |
| "arm": "w4096", |
| "brier": 0.32261680428571426, |
| "ece": 0.06391125541125536, |
| "n": 231, |
| "n_correct": 176, |
| "schema_validity_strict": 1.0, |
| "served": "these files", |
| "window": 4096 |
| } |
| ], |
| "name": "hobson-2b-v21", |
| "role": "final", |
| "run_id": "v21b-s5" |
| } |
