Spaces:
Running
Running
{
"numbers": {
"environments": 32,
"task_suites": 47,
"community_environments": 20,
"episodes": 1177,
"beam_episodes": 640,
"small_episodes": 240,
"beam_output_tokens": 1572150,
"suites_played_by": {
"null": 47,
"random": 14,
"small": 10,
"beam-low": 46,
"beam-medium": 6,
"beam-high": 12,
"beam-max": 1
}
},
"verdicts": [
{
"suite": "chess_env",
"env": "chess_env",
"notes": "vs moonfish depth 2; +1 win, -1 loss; capped at 30 moves",
"episodes": 14,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": -0.8333333333333334,
"std": 0.37267799624996495,
"within_task_std": 0.23570226039551584,
"grpo_signal": 0.5,
"out_tokens": 49037,
"invalid_steps": 0,
"steps": 108,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 3
},
"null": {
"episodes": 2,
"mean": -1.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 18,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 1
},
"random": {
"episodes": 6,
"mean": -1.0,
"std": 0.0,
"within_task_std": 0.0,
"grpo_signal": 0.0,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 100,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 3
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_solve_rate": 0.0,
"floor_policy": "null"
},
"statistical.reward_reachability": {
"pass": false,
"reference": "beam-low",
"pass@1": 0.0,
"pass@k": 0.0,
"k": 3
},
"statistical.difficulty_separation": {
"pass": false,
"reference_minus_floor": 0.16666666666666663,
"ci95": [
0.0,
0.5
],
"p": 0.33325,
"n": [
6,
2
]
},
"statistical.headroom": {
"pass": true,
"best_mean": -0.8333333333333334,
"max": 1.0,
"room_fraction": 0.9166666666666667
},
"statistical.variance_bound": {
"pass": false,
"within_task_std": 0.23570226039551584,
"signal": 0.16666666666666663
},
"statistical.grpo_signal": {
"pass": true,
"reference": "beam-low",
"share": 0.5,
"threshold": 0.2
}
},
"verdict": "out of reach: no player tested ever wins, Beam included",
"group": "reach",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 2,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": -1.0,
"solved": 0.0,
"grpo": null,
"n": 2
},
{
"policy": "random",
"x": 0.0,
"mean": -1.0,
"solved": 0.0,
"grpo": 0.0,
"n": 6
},
{
"policy": "beam-low",
"x": 0.0833,
"mean": -0.8333333333333334,
"solved": 0.0,
"grpo": 0.5,
"n": 6
}
],
"top_known": true
}
},
{
"suite": "connect4_env",
"env": "connect4_env",
"notes": "as shipped: no opponent is ever used",
"episodes": 21,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 9,
"mean": 1.0,
"std": 0.0,
"within_task_std": 0.0,
"grpo_signal": 0.0,
"out_tokens": 124935,
"invalid_steps": 0,
"steps": 151,
"pass@1": 1.0,
"pass@k": 1.0,
"k": 3
},
"null": {
"episodes": 3,
"mean": 1.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 57,
"pass@1": 1.0,
"pass@k": 1.0,
"k": 1
},
"random": {
"episodes": 9,
"mean": 1.0,
"std": 0.0,
"within_task_std": 0.0,
"grpo_signal": 0.0,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 206,
"pass@1": 1.0,
"pass@k": 1.0,
"k": 3
}
},
"checks": {
"semantic.floor_gap": {
"pass": false,
"floor_solve_rate": 1.0
}
},
"verdict": "broken as shipped: doing nothing already earns the top reward",
"defect": "No opponent: the agent moves for both sides, so every finished game pays +1.",
"group": "broken",
"rail": {
"rungs": [
{
"policy": "null",
"x": 1.0,
"mean": 1.0,
"solved": 1.0,
"grpo": null,
"n": 3
},
{
"policy": "random",
"x": 1.0,
"mean": 1.0,
"solved": 1.0,
"grpo": 0.0,
"n": 9
},
{
"policy": "beam-low",
"x": 1.0,
"mean": 1.0,
"solved": 1.0,
"grpo": 0.0,
"n": 9
}
],
"top_known": true
}
},
{
"suite": "connect4_env-patched",
"env": "connect4_env",
"notes": "with Beam's fix: a built-in opponent, +1 only for an agent win",
"episodes": 21,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 9,
"mean": 0.7777777777777778,
"std": 0.6285393610547089,
"within_task_std": 0.31426968052735443,
"grpo_signal": 0.3333333333333333,
"out_tokens": 43094,
"invalid_steps": 0,
"steps": 62,
"pass@1": 0.8888888888888888,
"pass@k": 1.0,
"k": 3
},
"null": {
"episodes": 3,
"mean": -1.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 15,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 1
},
"random": {
"episodes": 9,
"mean": -0.7777777777777778,
"std": 0.6285393610547089,
"within_task_std": 0.31426968052735443,
"grpo_signal": 0.3333333333333333,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 56,
"pass@1": 0.1111111111111111,
"pass@k": 0.3333333333333333,
"k": 3
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_solve_rate": 0.1111111111111111,
"floor_policy": "random"
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-low",
"pass@1": 0.8888888888888888,
"pass@k": 1.0,
"k": 3
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 1.5555555555555556,
"ci95": [
0.8888888888888888,
2.0
],
"p": 0.0,
"n": [
9,
9
]
},
"statistical.headroom": {
"pass": true,
"best_mean": 0.7777777777777778,
"max": 1.0,
"room_fraction": 0.1111111111111111
},
"statistical.variance_bound": {
"pass": true,
"within_task_std": 0.31426968052735443,
"signal": 1.5555555555555556
},
"statistical.grpo_signal": {
"pass": true,
"reference": "beam-low",
"share": 0.3333333333333333,
"threshold": 0.2
}
},
"verdict": "learnable for Beam; not yet tested with a small model",
"group": "worth",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": -1.0,
"solved": 0.0,
"grpo": null,
"n": 3
},
{
"policy": "random",
"x": 0.1111,
"mean": -0.7777777777777778,
"solved": 0.1111111111111111,
"grpo": 0.3333333333333333,
"n": 9
},
{
"policy": "beam-low",
"x": 0.8889,
"mean": 0.7777777777777778,
"solved": 0.8888888888888888,
"grpo": 0.3333333333333333,
"n": 9
}
],
"top_known": true
}
},
{
"suite": "dipg_safety_env",
"env": "dipg_safety_env",
"notes": "single medical-safety answer",
"episodes": 16,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 12,
"mean": -10.0,
"std": 0.0,
"within_task_std": 0.0,
"grpo_signal": 0.0,
"out_tokens": 2644,
"invalid_steps": 0,
"steps": 12
},
"null": {
"episodes": 4,
"mean": -10.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 4
}
},
"checks": {
"semantic.floor_gap": {
"pass": false,
"floor_mean": -10.0,
"top": -10.0,
"gap_fraction": 0.0
},
"statistical.reward_reachability": {
"pass": false,
"reference": "beam-low",
"gain_over_floor": 0.0,
"ci95": [
0.0,
0.0
],
"p": 1.0
},
"statistical.difficulty_separation": {
"pass": false,
"reference_minus_floor": 0.0,
"ci95": [
0.0,
0.0
],
"p": 1.0,
"n": [
12,
4
]
},
"statistical.variance_bound": {
"pass": true,
"within_task_std": 0.0,
"signal": 1e-09
},
"statistical.grpo_signal": {
"pass": false,
"reference": "beam-low",
"share": 0.0,
"threshold": 0.2
}
},
"verdict": "broken as shipped: its default answer markers are chat-control tokens for Harmony-format models such as Beam, so no answer clears the format gate",
"defect": "Its default answer markers are chat-control tokens for Harmony-format models such as Beam, so no answer clears the format gate.",
"group": "broken",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 4,
"provisional": false,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": -10.0,
"solved": null,
"grpo": null,
"n": 4
},
{
"policy": "beam-low",
"x": 0.0,
"mean": -10.0,
"solved": null,
"grpo": 0.0,
"n": 12
}
],
"top_known": false
}
},
{
"suite": "fraud_triage_env",
"env": "fraud_triage_env",
"notes": "episode is 200 transactions; capped at 15 and scored per step",
"episodes": 14,
"errors": 1,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 5,
"mean": 0.1346666666666667,
"std": 0.11174574314537049,
"within_task_std": 0.07285393610547089,
"grpo_signal": 1.0,
"out_tokens": 9350,
"invalid_steps": 0,
"steps": 75
},
"null": {
"episodes": 2,
"mean": -0.31333333333333335,
"std": 3.925231146709438e-17,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 30
},
"random": {
"episodes": 6,
"mean": -0.3422222222222222,
"std": 0.15499900437752714,
"within_task_std": 0.13699590792373983,
"grpo_signal": 1.0,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 90
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_mean": -0.31333333333333335,
"top": 0.22000000000000006,
"gap_fraction": 0.6896551724137933
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-low",
"gain_over_floor": 0.44800000000000006,
"ci95": [
0.33333333333333337,
0.5173333333333334
],
"p": 0.0
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 0.44800000000000006,
"ci95": [
0.33333333333333337,
0.5173333333333334
],
"p": 0.0,
"n": [
5,
2
]
},
"statistical.variance_bound": {
"pass": true,
"within_task_std": 0.07285393610547089,
"signal": 0.44800000000000006
},
"statistical.grpo_signal": {
"pass": true,
"reference": "beam-low",
"share": 1.0,
"threshold": 0.2
}
},
"verdict": "learnable for Beam; not yet tested with a small model",
"group": "worth",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 2,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.3103,
"mean": -0.31333333333333335,
"solved": null,
"grpo": null,
"n": 2
},
{
"policy": "random",
"x": 0.273,
"mean": -0.3422222222222222,
"solved": null,
"grpo": 1.0,
"n": 6
},
{
"policy": "beam-low",
"x": 0.8897,
"mean": 0.1346666666666667,
"solved": null,
"grpo": 1.0,
"n": 5
}
],
"top_known": false
}
},
{
"suite": "grid_world_env",
"env": "grid_world_env",
"notes": "5x5 grid, reach the goal",
"episodes": 14,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": 0.30000000000000004,
"std": 0.0,
"within_task_std": 0.0,
"grpo_signal": 0.0,
"out_tokens": 3845,
"invalid_steps": 0,
"steps": 48
},
"null": {
"episodes": 2,
"mean": -1.2,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 24
},
"random": {
"episodes": 6,
"mean": -1.2,
"std": 0.0,
"within_task_std": 0.0,
"grpo_signal": 0.0,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 72
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_mean": -1.2,
"top": 0.30000000000000004,
"gap_fraction": 1.0
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-low",
"gain_over_floor": 1.5,
"ci95": [
1.5,
1.5
],
"p": 0.0
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 1.5,
"ci95": [
1.5,
1.5
],
"p": 0.0,
"n": [
6,
2
]
},
"statistical.variance_bound": {
"pass": true,
"within_task_std": 0.0,
"signal": 1.5
},
"statistical.grpo_signal": {
"pass": false,
"reference": "beam-low",
"share": 0.0,
"threshold": 0.2
}
},
"verdict": "no gradient for Beam: its repeated attempts at a task almost always agree",
"group": "nograd",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 2,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": -1.2,
"solved": null,
"grpo": null,
"n": 2
},
{
"policy": "random",
"x": 0.0,
"mean": -1.2,
"solved": null,
"grpo": 0.0,
"n": 6
},
{
"policy": "beam-low",
"x": 1.0,
"mean": 0.30000000000000004,
"solved": null,
"grpo": 0.0,
"n": 6
}
],
"top_known": false
}
},
{
"suite": "hub-Bb-03__os_expert_env",
"env": "Bb-03/os_expert_env",
"notes": "Diagnose and resolve a production Linux failure in the forensic sandbox.",
"episodes": 9,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": 0.2933333333333333,
"std": 2.033926143092604,
"within_task_std": 0.9166666666666666,
"grpo_signal": 0.3333333333333333,
"out_tokens": 25213,
"invalid_steps": 0,
"steps": 90
},
"null": {
"episodes": 3,
"mean": -0.6233333333333334,
"std": 0.05185449728701348,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 45
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_mean": -0.6233333333333334,
"top": 4.84,
"gap_fraction": 0.9933333333333333
},
"statistical.reward_reachability": {
"pass": false,
"reference": "beam-low",
"gain_over_floor": 0.9166666666666667,
"ci95": [
-0.05499999999999994,
2.75
],
"p": 0.16275
},
"statistical.difficulty_separation": {
"pass": false,
"reference_minus_floor": 0.9166666666666667,
"ci95": [
-0.05499999999999994,
2.75
],
"p": 0.17075,
"n": [
6,
3
]
},
"statistical.variance_bound": {
"pass": true,
"within_task_std": 0.9166666666666666,
"signal": 0.9166666666666667
},
"statistical.grpo_signal": {
"pass": true,
"reference": "beam-low",
"share": 0.3333333333333333,
"threshold": 0.2
}
},
"verdict": "out of reach: Beam scores no better than the floor",
"group": "reach",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0067,
"mean": -0.6233333333333334,
"solved": null,
"grpo": null,
"n": 3
},
{
"policy": "beam-low",
"x": 0.1733,
"mean": 0.2933333333333333,
"solved": null,
"grpo": 0.3333333333333333,
"n": 6
}
],
"top_known": false
}
},
{
"suite": "hub-Boobesh007__code-review",
"env": "Boobesh007/code-review",
"notes": "Review a code snippet and identify bugs, security vulnerabilities, and quality issues.",
"episodes": 9,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": 0.715,
"std": 0.15955667749528168,
"within_task_std": 0.11499999999999999,
"grpo_signal": 1.0,
"out_tokens": 1029,
"invalid_steps": 0,
"steps": 6
},
"null": {
"episodes": 3,
"mean": 0.3,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 3
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_mean": 0.3,
"top": 0.99,
"gap_fraction": 1.0
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-low",
"gain_over_floor": 0.415,
"ci95": [
0.2916666666666667,
0.5366666666666668
],
"p": 0.0
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 0.415,
"ci95": [
0.2916666666666667,
0.5366666666666668
],
"p": 0.0,
"n": [
6,
3
]
},
"statistical.headroom": {
"pass": true,
"best_mean": 0.715,
"max": 0.99,
"room_fraction": 0.3985507246376812
},
"statistical.variance_bound": {
"pass": true,
"within_task_std": 0.11499999999999999,
"signal": 0.415
},
"statistical.grpo_signal": {
"pass": true,
"reference": "beam-low",
"share": 1.0,
"threshold": 0.2
}
},
"verdict": "learnable for Beam; not yet tested with a small model",
"group": "worth",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": 0.3,
"solved": null,
"grpo": null,
"n": 3
},
{
"policy": "beam-low",
"x": 0.6014,
"mean": 0.715,
"solved": null,
"grpo": 1.0,
"n": 6
}
],
"top_known": true
}
},
{
"suite": "hub-FineEnvs__geoguesser-env",
"env": "FineEnvs/geoguesser-env",
"notes": "Determine where the agent was dropped and commit a final coordinate guess.",
"episodes": 9,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": 0.0,
"std": 0.0,
"within_task_std": 0.0,
"grpo_signal": 0.0,
"out_tokens": 7631,
"invalid_steps": 0,
"steps": 41
},
"null": {
"episodes": 3,
"mean": 0.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 45
}
},
"checks": {
"semantic.floor_gap": {
"pass": false,
"floor_mean": 0.0,
"top": 0.0,
"gap_fraction": 0.0
},
"statistical.reward_reachability": {
"pass": false,
"reference": "beam-low",
"gain_over_floor": 0.0,
"ci95": [
0.0,
0.0
],
"p": 1.0
},
"statistical.difficulty_separation": {
"pass": false,
"reference_minus_floor": 0.0,
"ci95": [
0.0,
0.0
],
"p": 1.0,
"n": [
6,
3
]
},
"statistical.variance_bound": {
"pass": true,
"within_task_std": 0.0,
"signal": 1e-09
},
"statistical.grpo_signal": {
"pass": false,
"reference": "beam-low",
"share": 0.0,
"threshold": 0.2
}
},
"verdict": "flat: every player tested, Beam included, got the same reward; needs a look",
"group": "flat",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": 0.0,
"solved": null,
"grpo": null,
"n": 3
},
{
"policy": "beam-low",
"x": 0.0,
"mean": 0.0,
"solved": null,
"grpo": 0.0,
"n": 6
}
],
"top_known": false
}
},
{
"suite": "hub-FineEnvs__nayana-ocr-env",
"env": "FineEnvs/nayana-ocr-env",
"notes": "Read a page or crop, then transcribe, detect layout, or answer a question about it and receive a score.",
"episodes": 9,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": 0.16666666666666666,
"std": 0.372677996249965,
"within_task_std": 0.16666666666666666,
"grpo_signal": 0.3333333333333333,
"out_tokens": 1571,
"invalid_steps": 0,
"steps": 6
},
"null": {
"episodes": 3,
"mean": 0.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 3
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_mean": 0.0,
"top": 1.0,
"gap_fraction": 1.0
},
"statistical.reward_reachability": {
"pass": false,
"reference": "beam-low",
"gain_over_floor": 0.16666666666666666,
"ci95": [
0.0,
0.5
],
"p": 0.31775
},
"statistical.difficulty_separation": {
"pass": false,
"reference_minus_floor": 0.16666666666666666,
"ci95": [
0.0,
0.5
],
"p": 0.34075,
"n": [
6,
3
]
},
"statistical.headroom": {
"pass": true,
"best_mean": 0.16666666666666666,
"max": 1.0,
"room_fraction": 0.8333333333333334
},
"statistical.variance_bound": {
"pass": false,
"within_task_std": 0.16666666666666666,
"signal": 0.16666666666666666
},
"statistical.grpo_signal": {
"pass": true,
"reference": "beam-low",
"share": 0.3333333333333333,
"threshold": 0.2
}
},
"verdict": "out of reach: Beam scores no better than the floor",
"group": "reach",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": 0.0,
"solved": null,
"grpo": null,
"n": 3
},
{
"policy": "beam-low",
"x": 0.1667,
"mean": 0.16666666666666666,
"solved": null,
"grpo": 0.3333333333333333,
"n": 6
}
],
"top_known": true
}
},
{
"suite": "hub-RavichandraNayakar__customer_support_env",
"env": "RavichandraNayakar/customer_support_env",
"notes": "Resolve a customer support ticket through the three-phase multi-agent bidding, execution, and resolution protocol.",
"episodes": 9,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": 0.0,
"std": 0.0,
"within_task_std": 0.0,
"grpo_signal": 0.0,
"out_tokens": 18302,
"invalid_steps": 0,
"steps": 60,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 2
},
"null": {
"episodes": 3,
"mean": -0.5,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 3,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 1
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_solve_rate": 0.0,
"floor_policy": "null"
},
"statistical.reward_reachability": {
"pass": false,
"reference": "beam-low",
"pass@1": 0.0,
"pass@k": 0.0,
"k": 2
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 0.5,
"ci95": [
0.5,
0.5
],
"p": 0.0,
"n": [
6,
3
]
},
"statistical.headroom": {
"pass": true,
"best_mean": 0.0,
"max": 1.0,
"room_fraction": 0.6666666666666666
},
"statistical.variance_bound": {
"pass": true,
"within_task_std": 0.0,
"signal": 0.5
},
"statistical.grpo_signal": {
"pass": false,
"reference": "beam-low",
"share": 0.0,
"threshold": 0.2
}
},
"verdict": "out of reach: no player tested ever wins, Beam included",
"group": "reach",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": -0.5,
"solved": 0.0,
"grpo": null,
"n": 3
},
{
"policy": "beam-low",
"x": 0.3333,
"mean": 0.0,
"solved": 0.0,
"grpo": 0.0,
"n": 6
}
],
"top_known": true
}
},
{
"suite": "hub-TejasS1233__openenv-food-delivery-dispatch",
"env": "TejasS1233/openenv-food-delivery-dispatch",
"notes": "Dispatch couriers to orders and manage deliveries to maximize service quality and efficiency.",
"episodes": 9,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": 2.343,
"std": 0.0,
"within_task_std": 0.0,
"grpo_signal": 0.0,
"out_tokens": 9073,
"invalid_steps": 0,
"steps": 90
},
"null": {
"episodes": 3,
"mean": -8.039999999999997,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 45
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_mean": -8.039999999999997,
"top": 2.343,
"gap_fraction": 1.0
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-low",
"gain_over_floor": 10.382999999999997,
"ci95": [
10.382999999999997,
10.382999999999997
],
"p": 0.0
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 10.382999999999997,
"ci95": [
10.382999999999997,
10.382999999999997
],
"p": 0.0,
"n": [
6,
3
]
},
"statistical.variance_bound": {
"pass": true,
"within_task_std": 0.0,
"signal": 10.382999999999997
},
"statistical.grpo_signal": {
"pass": false,
"reference": "beam-low",
"share": 0.0,
"threshold": 0.2
}
},
"verdict": "no gradient for Beam: its repeated attempts at a task almost always agree",
"declared_max_wrong": {
"declared": 1.0,
"seen": 2.343
},
"group": "nograd",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": -8.039999999999997,
"solved": null,
"grpo": null,
"n": 3
},
{
"policy": "beam-low",
"x": 1.0,
"mean": 2.343,
"solved": null,
"grpo": 0.0,
"n": 6
}
],
"top_known": false
}
},
{
"suite": "hub-Vivaan13__smartinbox-pro",
"env": "Vivaan13/smartinbox-pro",
"notes": "Classify each incoming email as spam, normal, or urgent.",
"episodes": 9,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": 0.3499999999999999,
"std": 0.1773649721149397,
"within_task_std": 0.05833333333333334,
"grpo_signal": 0.6666666666666666,
"out_tokens": 8038,
"invalid_steps": 0,
"steps": 60
},
"null": {
"episodes": 3,
"mean": 0.34166666666666673,
"std": 0.042491829279939886,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 30
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_mean": 0.34166666666666673,
"top": 0.55,
"gap_fraction": 0.39682539682539675
},
"statistical.reward_reachability": {
"pass": false,
"reference": "beam-low",
"gain_over_floor": 0.008333333333333193,
"ci95": [
-0.15,
0.15000000000000008
],
"p": 0.45175
},
"statistical.difficulty_separation": {
"pass": false,
"reference_minus_floor": 0.008333333333333193,
"ci95": [
-0.14583333333333334,
0.15416666666666673
],
"p": 0.44175,
"n": [
6,
3
]
},
"statistical.variance_bound": {
"pass": false,
"within_task_std": 0.05833333333333334,
"signal": 0.008333333333333193
},
"statistical.grpo_signal": {
"pass": true,
"reference": "beam-low",
"share": 0.6666666666666666,
"threshold": 0.2
}
},
"verdict": "out of reach: Beam scores no better than the floor",
"group": "reach",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.6032,
"mean": 0.34166666666666673,
"solved": null,
"grpo": null,
"n": 3
},
{
"policy": "beam-low",
"x": 0.619,
"mean": 0.3499999999999999,
"solved": null,
"grpo": 0.6666666666666666,
"n": 6
}
],
"top_known": false
}
},
{
"suite": "hub-adishu__auto_jailbreak",
"env": "adishu/auto_jailbreak",
"notes": "Generate attack prompts that jailbreak the target LLM into revealing protected secrets or producing the forbidden output.",
"episodes": 9,
"errors": 6,
"harness_errors": 0,
"tiers": {
"null": {
"episodes": 3,
"mean": 0.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 30
}
},
"checks": {},
"verdict": "broken: most episodes error",
"group": "broken",
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": 0.0,
"solved": null,
"grpo": null,
"n": 3
}
],
"top_known": true
}
},
{
"suite": "hub-anugrahteesdollar__drugenv",
"env": "anugrahteesdollar/drugenv",
"notes": "Investigate a proposed drug target and submit a calibrated go or no-go validation report.",
"episodes": 9,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": 5.697989734708222,
"std": 0.402571107206282,
"within_task_std": 0.22788196528466273,
"grpo_signal": 1.0,
"out_tokens": 5425,
"invalid_steps": 0,
"steps": 41
},
"null": {
"episodes": 3,
"mean": -1.0629613547546677,
"std": 0.0012281489903368888,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 45
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_mean": -1.0629613547546677,
"top": 6.288936170212766,
"gap_fraction": 0.9998818903682777
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-low",
"gain_over_floor": 6.76095108946289,
"ci95": [
6.457033927490856,
7.096578924247419
],
"p": 0.0
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 6.76095108946289,
"ci95": [
6.465177816029766,
7.081847249062861
],
"p": 0.0,
"n": [
6,
3
]
},
"statistical.variance_bound": {
"pass": true,
"within_task_std": 0.22788196528466273,
"signal": 6.76095108946289
},
"statistical.grpo_signal": {
"pass": true,
"reference": "beam-low",
"share": 1.0,
"threshold": 0.2
}
},
"verdict": "learnable for Beam; not yet tested with a small model",
"declared_max_wrong": {
"declared": 1.0,
"seen": 6.288936170212766
},
"group": "worth",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0001,
"mean": -1.0629613547546677,
"solved": null,
"grpo": null,
"n": 3
},
{
"policy": "beam-low",
"x": 0.9196,
"mean": 5.697989734708222,
"solved": null,
"grpo": 1.0,
"n": 6
}
],
"top_known": false
}
},
{
"suite": "hub-creovateHQ__BrowserForge",
"env": "creovateHQ/BrowserForge",
"notes": "Click the no button on the page.",
"episodes": 9,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": 10.9,
"std": 0.0,
"within_task_std": 0.0,
"grpo_signal": 0.0,
"out_tokens": 394,
"invalid_steps": 0,
"steps": 6
},
"null": {
"episodes": 3,
"mean": -2.1,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 18
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_mean": -2.1,
"top": 10.9,
"gap_fraction": 1.0
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-low",
"gain_over_floor": 13.0,
"ci95": [
13.0,
13.0
],
"p": 0.0
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 13.0,
"ci95": [
13.0,
13.0
],
"p": 0.0,
"n": [
6,
3
]
},
"statistical.variance_bound": {
"pass": true,
"within_task_std": 0.0,
"signal": 13.0
},
"statistical.grpo_signal": {
"pass": false,
"reference": "beam-low",
"share": 0.0,
"threshold": 0.2
}
},
"verdict": "no gradient for Beam: its repeated attempts at a task almost always agree",
"group": "nograd",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": -2.1,
"solved": null,
"grpo": null,
"n": 3
},
{
"policy": "beam-low",
"x": 1.0,
"mean": 10.9,
"solved": null,
"grpo": 0.0,
"n": 6
}
],
"top_known": false
}
},
{
"suite": "hub-mykrex__webyx-env",
"env": "mykrex/webyx-env",
"notes": "Audit and fix HTML WCAG accessibility violations in an episode.",
"episodes": 9,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": 1.7666666666666668,
"std": 0.5436502143433366,
"within_task_std": 0.0,
"grpo_signal": 0.0,
"out_tokens": 4061,
"invalid_steps": 0,
"steps": 32
},
"null": {
"episodes": 3,
"mean": -0.7999999999999999,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 24
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_mean": -0.7999999999999999,
"top": 2.5000000000000004,
"gap_fraction": 1.0
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-low",
"gain_over_floor": 2.566666666666667,
"ci95": [
2.1333333333333333,
3.0
],
"p": 0.0
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 2.566666666666667,
"ci95": [
2.1333333333333333,
3.0
],
"p": 0.0,
"n": [
6,
3
]
},
"statistical.variance_bound": {
"pass": true,
"within_task_std": 0.0,
"signal": 2.566666666666667
},
"statistical.grpo_signal": {
"pass": false,
"reference": "beam-low",
"share": 0.0,
"threshold": 0.2
}
},
"verdict": "no gradient for Beam: its repeated attempts at a task almost always agree",
"declared_max_wrong": {
"declared": 1.0,
"seen": 2.5000000000000004
},
"group": "nograd",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": -0.7999999999999999,
"solved": null,
"grpo": null,
"n": 3
},
{
"policy": "beam-low",
"x": 0.7778,
"mean": 1.7666666666666668,
"solved": null,
"grpo": 0.0,
"n": 6
}
],
"top_known": false
}
},
{
"suite": "hub-openenv__atari_env",
"env": "openenv/atari_env",
"notes": "Play an Atari 2600 game by choosing legal actions to maximize the cumulative episode reward.",
"episodes": 18,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": 0.0,
"std": 0.0,
"within_task_std": 0.0,
"grpo_signal": 0.0,
"out_tokens": 23059,
"invalid_steps": 0,
"steps": 90
},
"null": {
"episodes": 3,
"mean": 0.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 45
},
"random": {
"episodes": 9,
"mean": 0.0,
"std": 0.0,
"within_task_std": 0.0,
"grpo_signal": 0.0,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 135
}
},
"checks": {
"semantic.floor_gap": {
"pass": false,
"floor_mean": 0.0,
"top": 0.0,
"gap_fraction": 0.0
},
"statistical.reward_reachability": {
"pass": false,
"reference": "beam-low",
"gain_over_floor": 0.0,
"ci95": [
0.0,
0.0
],
"p": 1.0
},
"statistical.difficulty_separation": {
"pass": false,
"reference_minus_floor": 0.0,
"ci95": [
0.0,
0.0
],
"p": 1.0,
"n": [
6,
3
]
},
"statistical.variance_bound": {
"pass": true,
"within_task_std": 0.0,
"signal": 1e-09
},
"statistical.grpo_signal": {
"pass": false,
"reference": "beam-low",
"share": 0.0,
"threshold": 0.2
}
},
"verdict": "flat: every player tested, Beam included, got the same reward; needs a look",
"group": "flat",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": 0.0,
"solved": null,
"grpo": null,
"n": 3
},
{
"policy": "random",
"x": 0.0,
"mean": 0.0,
"solved": null,
"grpo": 0.0,
"n": 9
},
{
"policy": "beam-low",
"x": 0.0,
"mean": 0.0,
"solved": null,
"grpo": 0.0,
"n": 6
}
],
"top_known": false
}
},
{
"suite": "hub-openenv__coding_env",
"env": "openenv/coding_env",
"notes": "Execute Python code and inspect its output.",
"episodes": 9,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": -0.09999999999999994,
"std": 1.2096831541082702,
"within_task_std": 0.5333333333333333,
"grpo_signal": 0.6666666666666666,
"out_tokens": 6007,
"invalid_steps": 0,
"steps": 30
},
"null": {
"episodes": 3,
"mean": 0.5,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 15
}
},
"checks": {
"semantic.floor_gap": {
"pass": false,
"floor_mean": 0.5,
"top": 0.5,
"gap_fraction": 0.0
},
"statistical.reward_reachability": {
"pass": false,
"reference": "beam-low",
"gain_over_floor": -0.6,
"ci95": [
-1.6833333333333333,
-0.016666666666666663
],
"p": 1.0
},
"statistical.difficulty_separation": {
"pass": false,
"reference_minus_floor": -0.6,
"ci95": [
-1.6833333333333333,
-0.016666666666666663
],
"p": 1.0,
"n": [
6,
3
]
},
"statistical.variance_bound": {
"pass": false,
"within_task_std": 0.5333333333333333,
"signal": 1e-09
},
"statistical.grpo_signal": {
"pass": true,
"reference": "beam-low",
"share": 0.6666666666666666,
"threshold": 0.2
}
},
"verdict": "weak signal: doing nothing already scores near the top",
"group": "weak",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 1.0,
"mean": 0.5,
"solved": null,
"grpo": null,
"n": 3
},
{
"policy": "beam-low",
"x": 0.8182,
"mean": -0.09999999999999994,
"solved": null,
"grpo": 0.6666666666666666,
"n": 6
}
],
"top_known": false
}
},
{
"suite": "hub-openenv__echo_env",
"env": "openenv/echo_env",
"notes": "Call echo tools to echo messages back from the environment.",
"episodes": 9,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": 0.0,
"std": 0.0,
"within_task_std": 0.0,
"grpo_signal": 0.0,
"out_tokens": 3531,
"invalid_steps": 0,
"steps": 18
},
"null": {
"episodes": 3,
"mean": 0.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 9
}
},
"checks": {
"semantic.floor_gap": {
"pass": false,
"floor_mean": 0.0,
"top": 0.0,
"gap_fraction": 0.0
},
"statistical.reward_reachability": {
"pass": false,
"reference": "beam-low",
"gain_over_floor": 0.0,
"ci95": [
0.0,
0.0
],
"p": 1.0
},
"statistical.difficulty_separation": {
"pass": false,
"reference_minus_floor": 0.0,
"ci95": [
0.0,
0.0
],
"p": 1.0,
"n": [
6,
3
]
},
"statistical.headroom": {
"pass": false,
"best_mean": 0.0,
"max": 0.0,
"room_fraction": 0.0
},
"statistical.variance_bound": {
"pass": true,
"within_task_std": 0.0,
"signal": 1e-09
},
"statistical.grpo_signal": {
"pass": false,
"reference": "beam-low",
"share": 0.0,
"threshold": 0.2
}
},
"verdict": "flat: every player tested, Beam included, got the same reward; needs a look",
"group": "flat",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": 0.0,
"solved": null,
"grpo": null,
"n": 3
},
{
"policy": "beam-low",
"x": 0.0,
"mean": 0.0,
"solved": null,
"grpo": 0.0,
"n": 6
}
],
"top_known": false
}
},
{
"suite": "hub-openenv__openspiel_env",
"env": "openenv/openspiel_env",
"notes": "Move horizontally to catch the falling ball.",
"episodes": 18,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": 0.0,
"std": 1.0,
"within_task_std": 0.3333333333333333,
"grpo_signal": 0.3333333333333333,
"out_tokens": 31019,
"invalid_steps": 0,
"steps": 54
},
"null": {
"episodes": 3,
"mean": -1.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 27
},
"random": {
"episodes": 9,
"mean": -0.3333333333333333,
"std": 0.9428090415820634,
"within_task_std": 0.6285393610547089,
"grpo_signal": 0.6666666666666666,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 81
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_mean": -0.3333333333333333,
"top": 1.0,
"gap_fraction": 0.6666666666666666
},
"statistical.reward_reachability": {
"pass": false,
"reference": "beam-low",
"gain_over_floor": 0.3333333333333333,
"ci95": [
-0.6666666666666666,
1.3333333333333333
],
"p": 0.3025
},
"statistical.difficulty_separation": {
"pass": false,
"reference_minus_floor": 0.3333333333333333,
"ci95": [
-0.6666666666666666,
1.3333333333333333
],
"p": 0.30525,
"n": [
6,
9
]
},
"statistical.headroom": {
"pass": true,
"best_mean": 0.0,
"max": 1.0,
"room_fraction": 0.5
},
"statistical.variance_bound": {
"pass": false,
"within_task_std": 0.3333333333333333,
"signal": 0.3333333333333333
},
"statistical.grpo_signal": {
"pass": true,
"reference": "beam-low",
"share": 0.3333333333333333,
"threshold": 0.2
}
},
"verdict": "out of reach: Beam scores no better than the floor",
"group": "reach",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": -1.0,
"solved": null,
"grpo": null,
"n": 3
},
{
"policy": "random",
"x": 0.3333,
"mean": -0.3333333333333333,
"solved": null,
"grpo": 0.6666666666666666,
"n": 9
},
{
"policy": "beam-low",
"x": 0.5,
"mean": 0.0,
"solved": null,
"grpo": 0.3333333333333333,
"n": 6
}
],
"top_known": true
}
},
{
"suite": "hub-sergiopaniego__repl",
"env": "sergiopaniego/repl",
"notes": "Execute Python code iteratively in a sandboxed REPL to analyze a provided context and produce a final answer.",
"episodes": 9,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": 0.725,
"std": 0.16266017746619277,
"within_task_std": 0.075,
"grpo_signal": 0.6666666666666666,
"out_tokens": 7972,
"invalid_steps": 0,
"steps": 51
},
"null": {
"episodes": 3,
"mean": 0.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 45
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_mean": 0.0,
"top": 1.0,
"gap_fraction": 1.0
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-low",
"gain_over_floor": 0.725,
"ci95": [
0.5916666666666668,
0.8583333333333334
],
"p": 0.0
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 0.725,
"ci95": [
0.6,
0.8583333333333334
],
"p": 0.0,
"n": [
6,
3
]
},
"statistical.headroom": {
"pass": true,
"best_mean": 0.725,
"max": 1.0,
"room_fraction": 0.275
},
"statistical.variance_bound": {
"pass": true,
"within_task_std": 0.075,
"signal": 0.725
},
"statistical.grpo_signal": {
"pass": true,
"reference": "beam-low",
"share": 0.6666666666666666,
"threshold": 0.2
}
},
"verdict": "learnable for Beam; not yet tested with a small model",
"group": "worth",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": 0.0,
"solved": null,
"grpo": null,
"n": 3
},
{
"policy": "beam-low",
"x": 0.725,
"mean": 0.725,
"solved": null,
"grpo": 0.6666666666666666,
"n": 6
}
],
"top_known": true
}
},
{
"suite": "hub-sergiopaniego__wordle",
"env": "sergiopaniego/wordle",
"notes": "Guess the secret 5-letter Wordle word within 6 attempts.",
"episodes": 9,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": 0.19444444444444442,
"std": 0.0921284663987611,
"within_task_std": 0.05555555555555556,
"grpo_signal": 0.6666666666666666,
"out_tokens": 14009,
"invalid_steps": 0,
"steps": 28
},
"null": {
"episodes": 3,
"mean": 0.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 6
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_mean": 0.0,
"top": 0.25,
"gap_fraction": 1.0
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-low",
"gain_over_floor": 0.19444444444444442,
"ci95": [
0.1111111111111111,
0.25
],
"p": 0.0
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 0.19444444444444442,
"ci95": [
0.1111111111111111,
0.25
],
"p": 0.0,
"n": [
6,
3
]
},
"statistical.variance_bound": {
"pass": true,
"within_task_std": 0.05555555555555556,
"signal": 0.19444444444444442
},
"statistical.grpo_signal": {
"pass": true,
"reference": "beam-low",
"share": 0.6666666666666666,
"threshold": 0.2
}
},
"verdict": "learnable for Beam; not yet tested with a small model",
"group": "worth",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": 0.0,
"solved": null,
"grpo": null,
"n": 3
},
{
"policy": "beam-low",
"x": 0.7778,
"mean": 0.19444444444444442,
"solved": null,
"grpo": 0.6666666666666666,
"n": 6
}
],
"top_known": false
}
},
{
"suite": "hub-souller__email-triage-env",
"env": "souller/email-triage-env",
"notes": "Triage emails by classifying, summarising, routing, and replying according to the selected task difficulty.",
"episodes": 9,
"errors": 3,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 3,
"mean": 0.6188888888888889,
"std": 0.23414673040216621,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 1528,
"invalid_steps": 0,
"steps": 6
},
"null": {
"episodes": 3,
"mean": 0.01,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 6
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_mean": 0.01,
"top": 0.99,
"gap_fraction": 1.0
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-low",
"gain_over_floor": 0.6088888888888889,
"ci95": [
0.32666666666666666,
0.9
],
"p": 0.0
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 0.6088888888888889,
"ci95": [
0.32666666666666666,
0.9
],
"p": 0.0,
"n": [
3,
3
]
},
"statistical.headroom": {
"pass": true,
"best_mean": 0.6188888888888889,
"max": 0.99,
"room_fraction": 0.37868480725623577
}
},
"verdict": "learnable for Beam; not yet tested with a small model",
"group": "worth",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": 0.01,
"solved": null,
"grpo": null,
"n": 3
},
{
"policy": "beam-low",
"x": 0.6213,
"mean": 0.6188888888888889,
"solved": null,
"grpo": null,
"n": 3
}
],
"top_known": true
}
},
{
"suite": "hub-yaswanth169__APISHIFT",
"env": "yaswanth169/APISHIFT",
"notes": "Migrate the client code to safely handle the breaking API changes in the scenario and submit a final migration plan.",
"episodes": 9,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": 0.01,
"std": 0.0,
"within_task_std": 0.0,
"grpo_signal": 0.0,
"out_tokens": 8493,
"invalid_steps": 0,
"steps": 36
},
"null": {
"episodes": 3,
"mean": 0.01,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 18
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_mean": 0.01,
"top": 0.99,
"gap_fraction": 1.0
},
"statistical.reward_reachability": {
"pass": false,
"reference": "beam-low",
"gain_over_floor": 0.0,
"ci95": [
0.0,
0.0
],
"p": 1.0
},
"statistical.difficulty_separation": {
"pass": false,
"reference_minus_floor": 0.0,
"ci95": [
0.0,
0.0
],
"p": 1.0,
"n": [
6,
3
]
},
"statistical.headroom": {
"pass": true,
"best_mean": 0.01,
"max": 0.99,
"room_fraction": 1.0
},
"statistical.variance_bound": {
"pass": true,
"within_task_std": 0.0,
"signal": 1e-09
},
"statistical.grpo_signal": {
"pass": false,
"reference": "beam-low",
"share": 0.0,
"threshold": 0.2
}
},
"verdict": "flat: every player tested, Beam included, got the same reward; needs a look",
"group": "flat",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": 0.01,
"solved": null,
"grpo": null,
"n": 3
},
{
"policy": "beam-low",
"x": 0.0,
"mean": 0.01,
"solved": null,
"grpo": 0.0,
"n": 6
}
],
"top_known": true
}
},
{
"suite": "hub-yaswanth169__data-janitor-env",
"env": "yaswanth169/data-janitor-env",
"notes": "Clean a messy tabular dataset by applying transformations so it matches the target schema and ground truth, then submit.",
"episodes": 9,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": 0.29466666666666663,
"std": 0.44212045744218725,
"within_task_std": 0.3377666666666667,
"grpo_signal": 1.0,
"out_tokens": 14193,
"invalid_steps": 0,
"steps": 55
},
"null": {
"episodes": 3,
"mean": 0.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 30
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_mean": 0.0,
"top": 1.0535,
"gap_fraction": 0.742476566354218
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-low",
"gain_over_floor": 0.29466666666666663,
"ci95": [
-0.04310000000000003,
0.6702833333333333
],
"p": 0.04425
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 0.29466666666666663,
"ci95": [
-0.044050000000000034,
0.6519333333333334
],
"p": 0.04725,
"n": [
6,
3
]
},
"statistical.variance_bound": {
"pass": false,
"within_task_std": 0.3377666666666667,
"signal": 0.29466666666666663
},
"statistical.grpo_signal": {
"pass": true,
"reference": "beam-low",
"share": 1.0,
"threshold": 0.2
}
},
"verdict": "learnable for Beam; not yet tested with a small model",
"declared_max_wrong": {
"declared": 1.0,
"seen": 1.0535
},
"group": "worth",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.2575,
"mean": 0.0,
"solved": null,
"grpo": null,
"n": 3
},
{
"policy": "beam-low",
"x": 0.4652,
"mean": 0.29466666666666663,
"solved": null,
"grpo": 1.0,
"n": 6
}
],
"top_known": false
}
},
{
"suite": "maze_env",
"env": "maze_env",
"notes": "8x8 maze; +10 at the exit, small penalties per move; capped at 25 moves",
"episodes": 24,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-high": {
"episodes": 3,
"mean": 9.35,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 79449,
"invalid_steps": 0,
"steps": 42,
"pass@1": 1.0,
"pass@k": 1.0,
"k": 1
},
"beam-low": {
"episodes": 9,
"mean": 9.283333333333335,
"std": 0.094280904158206,
"within_task_std": 0.06285393610547067,
"grpo_signal": 0.6666666666666666,
"out_tokens": 34180,
"invalid_steps": 0,
"steps": 138,
"pass@1": 1.0,
"pass@k": 1.0,
"k": 3
},
"null": {
"episodes": 3,
"mean": -5.849999999999999,
"std": 8.881784197001252e-16,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 75,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 1
},
"random": {
"episodes": 9,
"mean": -4.538888888888889,
"std": 0.40123266856150647,
"within_task_std": 0.3864308036286846,
"grpo_signal": 1.0,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 225,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 3
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_solve_rate": 0.0,
"floor_policy": "random"
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-high",
"pass@1": 1.0,
"pass@k": 1.0,
"k": 1
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 13.88888888888889,
"ci95": [
13.622222222222222,
14.155555555555555
],
"p": 0.0,
"n": [
3,
9
],
"between_beam_efforts": {
"stronger": "beam-high",
"weaker": "beam-low",
"diff": 0.06666666666666465,
"ci95": [
0.0,
0.13333333333333286
],
"p": 0.02825,
"separates": true
}
},
"statistical.headroom": {
"pass": false,
"best_mean": 9.35,
"max": 10.0,
"room_fraction": 0.0410094637223975
}
},
"verdict": "too easy for Beam; not yet tested with a small model",
"group": "easy",
"reference": "beam-high",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": -5.849999999999999,
"solved": 0.0,
"grpo": null,
"n": 3
},
{
"policy": "random",
"x": 0.0827,
"mean": -4.538888888888889,
"solved": 0.0,
"grpo": 1.0,
"n": 9
},
{
"policy": "beam-low",
"x": 0.9548,
"mean": 9.283333333333335,
"solved": 1.0,
"grpo": 0.6666666666666666,
"n": 9
},
{
"policy": "beam-high",
"x": 0.959,
"mean": 9.35,
"solved": 1.0,
"grpo": null,
"n": 3
}
],
"top_known": true
}
},
{
"suite": "os-2048",
"env": "openspiel_env",
"notes": "OpenSpiel 2048 (server default opponent)",
"episodes": 14,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": 60.0,
"std": 14.787382008545888,
"within_task_std": 13.412614930262478,
"grpo_signal": 1.0,
"out_tokens": 76422,
"invalid_steps": 0,
"steps": 90
},
"null": {
"episodes": 2,
"mean": 64.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 30
},
"random": {
"episodes": 6,
"mean": 54.666666666666664,
"std": 10.498677165349081,
"within_task_std": 6.599663291074444,
"grpo_signal": 1.0,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 90
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_mean": 64.0,
"top": 84.0,
"gap_fraction": 0.45454545454545453
},
"statistical.reward_reachability": {
"pass": false,
"reference": "beam-low",
"gain_over_floor": -4.0,
"ci95": [
-15.333333333333336,
7.350000000000056
],
"p": 0.762
},
"statistical.difficulty_separation": {
"pass": false,
"reference_minus_floor": -4.0,
"ci95": [
-15.333333333333336,
7.333333333333329
],
"p": 0.76525,
"n": [
6,
2
]
},
"statistical.variance_bound": {
"pass": false,
"within_task_std": 13.412614930262478,
"signal": 1e-09
},
"statistical.grpo_signal": {
"pass": true,
"reference": "beam-low",
"share": 1.0,
"threshold": 0.2
}
},
"verdict": "out of reach: Beam scores no better than the floor",
"group": "reach",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 2,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.5455,
"mean": 64.0,
"solved": null,
"grpo": null,
"n": 2
},
{
"policy": "random",
"x": 0.3333,
"mean": 54.666666666666664,
"solved": null,
"grpo": 1.0,
"n": 6
},
{
"policy": "beam-low",
"x": 0.4545,
"mean": 60.0,
"solved": null,
"grpo": 1.0,
"n": 6
}
],
"top_known": false
}
},
{
"suite": "os-blackjack",
"env": "openspiel_env",
"notes": "OpenSpiel blackjack (server default opponent)",
"episodes": 21,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 9,
"mean": -0.5555555555555556,
"std": 0.8314794192830981,
"within_task_std": 0.6285393610547089,
"grpo_signal": 0.6666666666666666,
"out_tokens": 4224,
"invalid_steps": 0,
"steps": 10,
"pass@1": 0.2222222222222222,
"pass@k": 0.6666666666666666,
"k": 3
},
"null": {
"episodes": 3,
"mean": -1.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 8,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 1
},
"random": {
"episodes": 9,
"mean": -0.1111111111111111,
"std": 0.9938079899999066,
"within_task_std": 0.31426968052735443,
"grpo_signal": 0.3333333333333333,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 15,
"pass@1": 0.4444444444444444,
"pass@k": 0.6666666666666666,
"k": 3
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_solve_rate": 0.4444444444444444,
"floor_policy": "random"
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-low",
"pass@1": 0.2222222222222222,
"pass@k": 0.6666666666666666,
"k": 3
},
"statistical.difficulty_separation": {
"pass": false,
"reference_minus_floor": -0.4444444444444445,
"ci95": [
-1.3333333333333333,
0.4444444444444445
],
"p": 0.8975,
"n": [
9,
9
]
},
"statistical.headroom": {
"pass": true,
"best_mean": -0.5555555555555556,
"max": 1.0,
"room_fraction": 0.7777777777777778
},
"statistical.variance_bound": {
"pass": false,
"within_task_std": 0.6285393610547089,
"signal": 1e-09
},
"statistical.grpo_signal": {
"pass": true,
"reference": "beam-low",
"share": 0.6666666666666666,
"threshold": 0.2
}
},
"verdict": "skill doesn't show yet: Beam isn't clearly above the floor in these games",
"group": "unsure",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": -1.0,
"solved": 0.0,
"grpo": null,
"n": 3
},
{
"policy": "random",
"x": 0.4444,
"mean": -0.1111111111111111,
"solved": 0.4444444444444444,
"grpo": 0.3333333333333333,
"n": 9
},
{
"policy": "beam-low",
"x": 0.2222,
"mean": -0.5555555555555556,
"solved": 0.2222222222222222,
"grpo": 0.6666666666666666,
"n": 9
}
],
"top_known": true
}
},
{
"suite": "os-catch",
"env": "openspiel_env",
"notes": "OpenSpiel catch (server default opponent)",
"episodes": 21,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 9,
"mean": -0.7777777777777778,
"std": 0.6285393610547089,
"within_task_std": 0.31426968052735443,
"grpo_signal": 0.3333333333333333,
"out_tokens": 43737,
"invalid_steps": 0,
"steps": 81,
"pass@1": 0.1111111111111111,
"pass@k": 0.3333333333333333,
"k": 3
},
"null": {
"episodes": 3,
"mean": -0.3333333333333333,
"std": 0.9428090415820634,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 27,
"pass@1": 0.3333333333333333,
"pass@k": 0.3333333333333333,
"k": 1
},
"random": {
"episodes": 9,
"mean": -0.7777777777777778,
"std": 0.6285393610547089,
"within_task_std": 0.31426968052735443,
"grpo_signal": 0.3333333333333333,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 81,
"pass@1": 0.1111111111111111,
"pass@k": 0.3333333333333333,
"k": 3
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_solve_rate": 0.3333333333333333,
"floor_policy": "null"
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-low",
"pass@1": 0.1111111111111111,
"pass@k": 0.3333333333333333,
"k": 3
},
"statistical.difficulty_separation": {
"pass": false,
"reference_minus_floor": -0.4444444444444445,
"ci95": [
-1.7777777777777777,
0.4444444444444444
],
"p": 0.8025,
"n": [
9,
3
]
},
"statistical.headroom": {
"pass": true,
"best_mean": -0.7777777777777778,
"max": 1.0,
"room_fraction": 0.8888888888888888
},
"statistical.variance_bound": {
"pass": false,
"within_task_std": 0.31426968052735443,
"signal": 1e-09
},
"statistical.grpo_signal": {
"pass": true,
"reference": "beam-low",
"share": 0.3333333333333333,
"threshold": 0.2
}
},
"verdict": "skill doesn't show yet: Beam isn't clearly above the floor in these games",
"group": "unsure",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.3333,
"mean": -0.3333333333333333,
"solved": 0.3333333333333333,
"grpo": null,
"n": 3
},
{
"policy": "random",
"x": 0.1111,
"mean": -0.7777777777777778,
"solved": 0.1111111111111111,
"grpo": 0.3333333333333333,
"n": 9
},
{
"policy": "beam-low",
"x": 0.1111,
"mean": -0.7777777777777778,
"solved": 0.1111111111111111,
"grpo": 0.3333333333333333,
"n": 9
}
],
"top_known": true
}
},
{
"suite": "os-cliff_walking",
"env": "openspiel_env",
"notes": "OpenSpiel cliff_walking (server default opponent)",
"episodes": 14,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": -100.5,
"std": 1.118033988749895,
"within_task_std": 0.7071067811865476,
"grpo_signal": 0.5,
"out_tokens": 1137,
"invalid_steps": 0,
"steps": 9
},
"null": {
"episodes": 2,
"mean": -100.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 2
},
"random": {
"episodes": 6,
"mean": -104.33333333333333,
"std": 3.2998316455372216,
"within_task_std": 2.250217458768164,
"grpo_signal": 1.0,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 32
}
},
"checks": {
"semantic.floor_gap": {
"pass": false,
"floor_mean": -100.0,
"top": -100.0,
"gap_fraction": 0.0
},
"statistical.reward_reachability": {
"pass": false,
"reference": "beam-low",
"gain_over_floor": -0.5,
"ci95": [
-1.5,
0.0
],
"p": 1.0
},
"statistical.difficulty_separation": {
"pass": false,
"reference_minus_floor": -0.5,
"ci95": [
-1.5,
0.0
],
"p": 1.0,
"n": [
6,
2
]
},
"statistical.variance_bound": {
"pass": false,
"within_task_std": 0.7071067811865476,
"signal": 1e-09
},
"statistical.grpo_signal": {
"pass": true,
"reference": "beam-low",
"share": 0.5,
"threshold": 0.2
}
},
"verdict": "weak signal: doing nothing already scores near the top",
"group": "weak",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 2,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 1.0,
"mean": -100.0,
"solved": null,
"grpo": null,
"n": 2
},
{
"policy": "random",
"x": 0.5667,
"mean": -104.33333333333333,
"solved": null,
"grpo": 1.0,
"n": 6
},
{
"policy": "beam-low",
"x": 0.95,
"mean": -100.5,
"solved": null,
"grpo": 0.5,
"n": 6
}
],
"top_known": false
}
},
{
"suite": "os-kuhn_poker",
"env": "openspiel_env",
"notes": "OpenSpiel kuhn_poker (server default opponent)",
"episodes": 21,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 9,
"mean": 0.7777777777777778,
"std": 1.0304020550550783,
"within_task_std": 0.7856742013183862,
"grpo_signal": 1.0,
"out_tokens": 5531,
"invalid_steps": 0,
"steps": 11
},
"null": {
"episodes": 3,
"mean": 0.3333333333333333,
"std": 0.9428090415820634,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 4
},
"random": {
"episodes": 9,
"mean": -0.5555555555555556,
"std": 1.4229164972072998,
"within_task_std": 1.0999438818457408,
"grpo_signal": 1.0,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 13
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_mean": 0.3333333333333333,
"top": 2.0,
"gap_fraction": 0.4166666666666667
},
"statistical.reward_reachability": {
"pass": false,
"reference": "beam-low",
"gain_over_floor": 0.4444444444444445,
"ci95": [
-0.7777777777777778,
1.7777777777777777
],
"p": 0.3025
},
"statistical.difficulty_separation": {
"pass": false,
"reference_minus_floor": 0.4444444444444445,
"ci95": [
-0.7777777777777778,
1.7777777777777777
],
"p": 0.29075,
"n": [
9,
3
]
},
"statistical.variance_bound": {
"pass": false,
"within_task_std": 0.7856742013183862,
"signal": 0.4444444444444445
},
"statistical.grpo_signal": {
"pass": true,
"reference": "beam-low",
"share": 1.0,
"threshold": 0.2
}
},
"verdict": "out of reach: Beam scores no better than the floor",
"group": "reach",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.5833,
"mean": 0.3333333333333333,
"solved": null,
"grpo": null,
"n": 3
},
{
"policy": "random",
"x": 0.3611,
"mean": -0.5555555555555556,
"solved": null,
"grpo": 1.0,
"n": 9
},
{
"policy": "beam-low",
"x": 0.6944,
"mean": 0.7777777777777778,
"solved": null,
"grpo": 1.0,
"n": 9
}
],
"top_known": false
}
},
{
"suite": "os-tic_tac_toe",
"env": "openspiel_env",
"notes": "OpenSpiel tic_tac_toe (server default opponent)",
"episodes": 24,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-high": {
"episodes": 3,
"mean": 1.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 39268,
"invalid_steps": 0,
"steps": 9,
"pass@1": 1.0,
"pass@k": 1.0,
"k": 1
},
"beam-low": {
"episodes": 9,
"mean": 0.8888888888888888,
"std": 0.3142696805273545,
"within_task_std": 0.15713484026367722,
"grpo_signal": 0.3333333333333333,
"out_tokens": 22468,
"invalid_steps": 0,
"steps": 35,
"pass@1": 0.8888888888888888,
"pass@k": 1.0,
"k": 3
},
"null": {
"episodes": 3,
"mean": 1.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 9,
"pass@1": 1.0,
"pass@k": 1.0,
"k": 1
},
"random": {
"episodes": 9,
"mean": 0.3333333333333333,
"std": 0.816496580927726,
"within_task_std": 0.4714045207910316,
"grpo_signal": 0.6666666666666666,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 39,
"pass@1": 0.5555555555555556,
"pass@k": 1.0,
"k": 3
}
},
"checks": {
"semantic.floor_gap": {
"pass": false,
"floor_solve_rate": 1.0,
"floor_policy": "null"
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-high",
"pass@1": 1.0,
"pass@k": 1.0,
"k": 1
},
"statistical.difficulty_separation": {
"pass": false,
"reference_minus_floor": 0.0,
"ci95": [
0.0,
0.0
],
"p": 1.0,
"n": [
3,
3
],
"between_beam_efforts": {
"stronger": "beam-high",
"weaker": "beam-low",
"diff": 0.11111111111111116,
"ci95": [
0.0,
0.33333333333333337
],
"p": 0.341,
"separates": false
}
},
"statistical.headroom": {
"pass": false,
"best_mean": 1.0,
"max": 1.0,
"room_fraction": 0.0
}
},
"verdict": "weak signal: a do-nothing or random policy already solves 100%",
"group": "weak",
"reference": "beam-high",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 1.0,
"mean": 1.0,
"solved": 1.0,
"grpo": null,
"n": 3
},
{
"policy": "random",
"x": 0.6667,
"mean": 0.3333333333333333,
"solved": 0.5555555555555556,
"grpo": 0.6666666666666666,
"n": 9
},
{
"policy": "beam-low",
"x": 0.9444,
"mean": 0.8888888888888888,
"solved": 0.8888888888888888,
"grpo": 0.3333333333333333,
"n": 9
},
{
"policy": "beam-high",
"x": 1.0,
"mean": 1.0,
"solved": 1.0,
"grpo": null,
"n": 3
}
],
"top_known": true
}
},
{
"suite": "pelican_svg_env",
"env": "pelican_svg_env",
"notes": "draw an SVG from a text prompt; layered score",
"episodes": 12,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 9,
"mean": 1.0,
"std": 0.0,
"within_task_std": 0.0,
"grpo_signal": 0.0,
"out_tokens": 11376,
"invalid_steps": 0,
"steps": 9
},
"null": {
"episodes": 3,
"mean": 0.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 3
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_mean": 0.0,
"top": 1.0,
"gap_fraction": 1.0
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-low",
"gain_over_floor": 1.0,
"ci95": [
1.0,
1.0
],
"p": 0.0
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 1.0,
"ci95": [
1.0,
1.0
],
"p": 0.0,
"n": [
9,
3
]
},
"statistical.headroom": {
"pass": false,
"best_mean": 1.0,
"max": 1.0,
"room_fraction": 0.0
},
"statistical.variance_bound": {
"pass": true,
"within_task_std": 0.0,
"signal": 1.0
},
"statistical.grpo_signal": {
"pass": false,
"reference": "beam-low",
"share": 0.0,
"threshold": 0.2
}
},
"verdict": "broken as shipped: the vision judge is off by default, so the reward checks only shapes and any two-wheeled drawing scores 1.00",
"defect": "The vision judge is off by default, so the reward checks only shapes and any two-wheeled drawing scores 1.00.",
"group": "broken",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 3,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": 0.0,
"solved": null,
"grpo": null,
"n": 3
},
{
"policy": "beam-low",
"x": 1.0,
"mean": 1.0,
"solved": null,
"grpo": 0.0,
"n": 9
}
],
"top_known": true
}
},
{
"suite": "rg-basic_arithmetic",
"env": "reasoning_gym_env",
"notes": "reasoning_gym family basic_arithmetic",
"episodes": 64,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-high": {
"episodes": 8,
"mean": 0.875,
"std": 0.33071891388307384,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 1674,
"invalid_steps": 0,
"steps": 8,
"pass@1": 0.875,
"pass@k": 0.875,
"k": 1
},
"beam-low": {
"episodes": 24,
"mean": 0.875,
"std": 0.33071891388307384,
"within_task_std": 0.0,
"grpo_signal": 0.0,
"out_tokens": 4075,
"invalid_steps": 0,
"steps": 24,
"pass@1": 0.875,
"pass@k": 0.875,
"k": 3
},
"null": {
"episodes": 8,
"mean": 0.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 8,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 1
},
"small": {
"episodes": 24,
"mean": 0.03888375600689673,
"std": 0.06005724529505568,
"within_task_std": 0.02657878796645543,
"grpo_signal": 0.5,
"out_tokens": 983,
"invalid_steps": 0,
"steps": 24,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 3
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_solve_rate": 0.0,
"floor_policy": "null"
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-high",
"pass@1": 0.875,
"pass@k": 0.875,
"k": 1
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 0.875,
"ci95": [
0.625,
1.0
],
"p": 0.0,
"n": [
8,
8
],
"between_beam_efforts": {
"stronger": "beam-high",
"weaker": "beam-high",
"diff": 0.0,
"ci95": [
-0.375,
0.375
],
"p": 0.655,
"separates": false
}
},
"statistical.headroom": {
"pass": true,
"best_mean": 0.875,
"max": 1.0,
"room_fraction": 0.125
},
"trainee.reachability": {
"pass": false,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 3
},
"trainee.grpo_signal": {
"pass": true,
"share": 0.5,
"threshold": 0.2
},
"trainee.rule": {
"prediction": "improves"
},
"trainee.room_below_reference": {
"pass": true,
"reference_minus_trainee": 0.8361162439931032,
"p": 0.0
},
"statistical.adversarial_floor": {
"pass": true,
"strategy": "constant",
"label": "always \"-2\"",
"answer": "-2",
"score": 0.013,
"learner_mean": 0.039,
"added_after_training": true
}
},
"verdict": "worth training the small model on",
"group": "worth",
"reference": "beam-high",
"trainee": "small",
"judged_for": "small model",
"tasks": 8,
"provisional": false,
"gameable": false,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": 0.0,
"solved": 0.0,
"grpo": null,
"n": 8
},
{
"policy": "small",
"x": 0.0389,
"mean": 0.03888375600689673,
"solved": 0.0,
"grpo": 0.5,
"n": 24
},
{
"policy": "beam-low",
"x": 0.875,
"mean": 0.875,
"solved": 0.875,
"grpo": 0.0,
"n": 24
},
{
"policy": "beam-high",
"x": 0.875,
"mean": 0.875,
"solved": 0.875,
"grpo": null,
"n": 8
}
],
"top_known": true
}
},
{
"suite": "rg-chain_sum",
"env": "reasoning_gym_env",
"notes": "reasoning_gym family chain_sum",
"episodes": 64,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-high": {
"episodes": 8,
"mean": 1.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 622,
"invalid_steps": 0,
"steps": 8,
"pass@1": 1.0,
"pass@k": 1.0,
"k": 1
},
"beam-low": {
"episodes": 24,
"mean": 1.0,
"std": 0.0,
"within_task_std": 0.0,
"grpo_signal": 0.0,
"out_tokens": 1749,
"invalid_steps": 0,
"steps": 24,
"pass@1": 1.0,
"pass@k": 1.0,
"k": 3
},
"null": {
"episodes": 8,
"mean": 0.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 8,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 1
},
"small": {
"episodes": 24,
"mean": 0.028928034934775193,
"std": 0.0486549908230185,
"within_task_std": 0.024506781643919416,
"grpo_signal": 0.625,
"out_tokens": 786,
"invalid_steps": 0,
"steps": 24,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 3
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_solve_rate": 0.0,
"floor_policy": "null"
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-high",
"pass@1": 1.0,
"pass@k": 1.0,
"k": 1
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 1.0,
"ci95": [
1.0,
1.0
],
"p": 0.0,
"n": [
8,
8
],
"between_beam_efforts": {
"stronger": "beam-high",
"weaker": "beam-high",
"diff": 0.0,
"ci95": [
0.0,
0.0
],
"p": 1.0,
"separates": false
}
},
"statistical.headroom": {
"pass": false,
"best_mean": 1.0,
"max": 1.0,
"room_fraction": 0.0
},
"trainee.reachability": {
"pass": false,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 3
},
"trainee.grpo_signal": {
"pass": true,
"share": 0.625,
"threshold": 0.2
},
"trainee.rule": {
"prediction": "improves"
},
"trainee.room_below_reference": {
"pass": true,
"reference_minus_trainee": 0.9710719650652249,
"p": 0.0
},
"statistical.adversarial_floor": {
"pass": true,
"strategy": "constant",
"label": "always \"5\"",
"answer": "5",
"score": 0.01,
"learner_mean": 0.029,
"added_after_training": true
}
},
"verdict": "worth training the small model on",
"group": "worth",
"reference": "beam-high",
"trainee": "small",
"judged_for": "small model",
"tasks": 8,
"provisional": false,
"gameable": false,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": 0.0,
"solved": 0.0,
"grpo": null,
"n": 8
},
{
"policy": "small",
"x": 0.0289,
"mean": 0.028928034934775193,
"solved": 0.0,
"grpo": 0.625,
"n": 24
},
{
"policy": "beam-low",
"x": 1.0,
"mean": 1.0,
"solved": 1.0,
"grpo": 0.0,
"n": 24
},
{
"policy": "beam-high",
"x": 1.0,
"mean": 1.0,
"solved": 1.0,
"grpo": null,
"n": 8
}
],
"top_known": true
}
},
{
"suite": "rg-countdown",
"env": "reasoning_gym_env",
"notes": "reasoning_gym family countdown",
"episodes": 72,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-high": {
"episodes": 8,
"mean": 1.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 41747,
"invalid_steps": 0,
"steps": 8,
"pass@1": 1.0,
"pass@k": 1.0,
"k": 1
},
"beam-low": {
"episodes": 24,
"mean": 0.6833333333333332,
"std": 0.44783429475148007,
"within_task_std": 0.16793786053180504,
"grpo_signal": 0.375,
"out_tokens": 27951,
"invalid_steps": 0,
"steps": 24,
"pass@1": 0.6666666666666666,
"pass@k": 0.875,
"k": 3
},
"beam-medium": {
"episodes": 8,
"mean": 0.7625,
"std": 0.41136206679760834,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 18989,
"invalid_steps": 0,
"steps": 8,
"pass@1": 0.75,
"pass@k": 0.75,
"k": 1
},
"null": {
"episodes": 8,
"mean": 0.01,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 8,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 1
},
"small": {
"episodes": 24,
"mean": 0.03833333333333334,
"std": 0.01818118685772619,
"within_task_std": 0.007071067811865479,
"grpo_signal": 0.375,
"out_tokens": 295,
"invalid_steps": 0,
"steps": 24,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 3
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_solve_rate": 0.0,
"floor_policy": "null"
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-high",
"pass@1": 1.0,
"pass@k": 1.0,
"k": 1
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 0.99,
"ci95": [
0.99,
0.99
],
"p": 0.0,
"n": [
8,
8
],
"between_beam_efforts": {
"stronger": "beam-high",
"weaker": "beam-low",
"diff": 0.31666666666666676,
"ci95": [
0.15833333333333321,
0.5145833333333334
],
"p": 0.0005,
"separates": true
}
},
"statistical.headroom": {
"pass": false,
"best_mean": 1.0,
"max": 1.0,
"room_fraction": 0.0
},
"trainee.reachability": {
"pass": false,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 3
},
"trainee.grpo_signal": {
"pass": true,
"share": 0.375,
"threshold": 0.2
},
"trainee.rule": {
"prediction": "improves"
},
"trainee.room_below_reference": {
"pass": true,
"reference_minus_trainee": 0.9616666666666667,
"p": 0.0
},
"statistical.adversarial_floor": {
"pass": true,
"strategy": "constant",
"label": "always one fixed answer",
"answer": "6*(98 - 54) - 34 - 66",
"score": 0.05,
"learner_mean": 0.038,
"added_after_training": true
}
},
"verdict": "worth training the small model on",
"group": "worth",
"reference": "beam-high",
"trainee": "small",
"judged_for": "small model",
"tasks": 8,
"provisional": false,
"gameable": false,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": 0.01,
"solved": 0.0,
"grpo": null,
"n": 8
},
{
"policy": "small",
"x": 0.0286,
"mean": 0.03833333333333334,
"solved": 0.0,
"grpo": 0.375,
"n": 24
},
{
"policy": "beam-low",
"x": 0.6801,
"mean": 0.6833333333333332,
"solved": 0.6666666666666666,
"grpo": 0.375,
"n": 24
},
{
"policy": "beam-medium",
"x": 0.7601,
"mean": 0.7625,
"solved": 0.75,
"grpo": null,
"n": 8
},
{
"policy": "beam-high",
"x": 1.0,
"mean": 1.0,
"solved": 1.0,
"grpo": null,
"n": 8
}
],
"top_known": true
}
},
{
"suite": "rg-game_of_life",
"env": "reasoning_gym_env",
"notes": "reasoning_gym family game_of_life",
"episodes": 80,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-high": {
"episodes": 8,
"mean": 0.99625,
"std": 0.009921567416492224,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 59220,
"invalid_steps": 0,
"steps": 8,
"pass@1": 0.875,
"pass@k": 0.875,
"k": 1
},
"beam-low": {
"episodes": 24,
"mean": 0.9775,
"std": 0.03467107343785787,
"within_task_std": 0.022835008406255256,
"grpo_signal": 1.0,
"out_tokens": 56359,
"invalid_steps": 0,
"steps": 24,
"pass@1": 0.375,
"pass@k": 0.875,
"k": 3
},
"beam-max": {
"episodes": 8,
"mean": 0.9975,
"std": 0.006614378277661483,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 69050,
"invalid_steps": 0,
"steps": 8,
"pass@1": 0.875,
"pass@k": 0.875,
"k": 1
},
"beam-medium": {
"episodes": 8,
"mean": 0.99875,
"std": 0.0033071891388307415,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 37490,
"invalid_steps": 0,
"steps": 8,
"pass@1": 0.875,
"pass@k": 0.875,
"k": 1
},
"null": {
"episodes": 8,
"mean": 0.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 8,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 1
},
"small": {
"episodes": 24,
"mean": 0.0,
"std": 0.0,
"within_task_std": 0.0,
"grpo_signal": 0.0,
"out_tokens": 1536,
"invalid_steps": 0,
"steps": 24,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 3
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_solve_rate": 0.0,
"floor_policy": "null"
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-medium",
"pass@1": 0.875,
"pass@k": 0.875,
"k": 1
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 0.99875,
"ci95": [
0.9962500000000001,
1.0
],
"p": 0.0,
"n": [
8,
8
],
"between_beam_efforts": {
"stronger": "beam-medium",
"weaker": "beam-low",
"diff": 0.02124999999999999,
"ci95": [
0.008739583333333266,
0.03708333333333336
],
"p": 0.0,
"separates": true
}
},
"statistical.headroom": {
"pass": false,
"best_mean": 0.99875,
"max": 1.0,
"room_fraction": 0.0012499999999999734
},
"trainee.reachability": {
"pass": false,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 3
},
"trainee.grpo_signal": {
"pass": false,
"share": 0.0,
"threshold": 0.2
},
"trainee.rule": {
"prediction": "too hard now"
},
"trainee.room_below_reference": {
"pass": true,
"reference_minus_trainee": 0.99875,
"p": 0.0
},
"statistical.adversarial_floor": {
"pass": false,
"strategy": "constant",
"label": "always an empty board (every cell dead)",
"answer": "[[0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0...",
"score": 0.962,
"learner_mean": 0.0,
"added_after_training": true
}
},
"verdict": "too hard for the small model now: it earns no reward, while Beam solves 88%",
"group": "hard",
"reference": "beam-medium",
"trainee": "small",
"judged_for": "small model",
"tasks": 8,
"provisional": false,
"gameable": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": 0.0,
"solved": 0.0,
"grpo": null,
"n": 8
},
{
"policy": "small",
"x": 0.0,
"mean": 0.0,
"solved": 0.0,
"grpo": 0.0,
"n": 24
},
{
"policy": "beam-low",
"x": 0.9775,
"mean": 0.9775,
"solved": 0.375,
"grpo": 1.0,
"n": 24
},
{
"policy": "beam-medium",
"x": 0.9988,
"mean": 0.99875,
"solved": 0.875,
"grpo": null,
"n": 8
},
{
"policy": "beam-high",
"x": 0.9962,
"mean": 0.99625,
"solved": 0.875,
"grpo": null,
"n": 8
},
{
"policy": "beam-max",
"x": 0.9975,
"mean": 0.9975,
"solved": 0.875,
"grpo": null,
"n": 8
}
],
"top_known": true
}
},
{
"suite": "rg-leg_counting",
"env": "reasoning_gym_env",
"notes": "reasoning_gym family leg_counting",
"episodes": 72,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-high": {
"episodes": 8,
"mean": 1.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 9648,
"invalid_steps": 0,
"steps": 8,
"pass@1": 1.0,
"pass@k": 1.0,
"k": 1
},
"beam-low": {
"episodes": 24,
"mean": 0.75,
"std": 0.4330127018922193,
"within_task_std": 0.23570226039551584,
"grpo_signal": 0.5,
"out_tokens": 6811,
"invalid_steps": 0,
"steps": 24,
"pass@1": 0.75,
"pass@k": 1.0,
"k": 3
},
"beam-medium": {
"episodes": 8,
"mean": 0.75,
"std": 0.4330127018922193,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 4137,
"invalid_steps": 0,
"steps": 8,
"pass@1": 0.75,
"pass@k": 0.75,
"k": 1
},
"null": {
"episodes": 8,
"mean": 0.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 8,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 1
},
"small": {
"episodes": 24,
"mean": 0.0,
"std": 0.0,
"within_task_std": 0.0,
"grpo_signal": 0.0,
"out_tokens": 1292,
"invalid_steps": 0,
"steps": 24,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 3
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_solve_rate": 0.0,
"floor_policy": "null"
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-high",
"pass@1": 1.0,
"pass@k": 1.0,
"k": 1
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 1.0,
"ci95": [
1.0,
1.0
],
"p": 0.0,
"n": [
8,
8
],
"between_beam_efforts": {
"stronger": "beam-high",
"weaker": "beam-low",
"diff": 0.25,
"ci95": [
0.08333333333333337,
0.41666666666666663
],
"p": 0.00075,
"separates": true
}
},
"statistical.headroom": {
"pass": false,
"best_mean": 1.0,
"max": 1.0,
"room_fraction": 0.0
},
"trainee.reachability": {
"pass": false,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 3
},
"trainee.grpo_signal": {
"pass": false,
"share": 0.0,
"threshold": 0.2
},
"trainee.rule": {
"prediction": "too hard now"
},
"trainee.room_below_reference": {
"pass": true,
"reference_minus_trainee": 1.0,
"p": 0.0
},
"statistical.adversarial_floor": {
"pass": true,
"strategy": "constant",
"label": "always \"184\"",
"answer": "184",
"score": 0.025,
"learner_mean": 0.0,
"added_after_training": true
}
},
"verdict": "too hard for the small model now: it earns no reward, while Beam solves 100%",
"group": "hard",
"reference": "beam-high",
"trainee": "small",
"judged_for": "small model",
"tasks": 8,
"provisional": false,
"gameable": false,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": 0.0,
"solved": 0.0,
"grpo": null,
"n": 8
},
{
"policy": "small",
"x": 0.0,
"mean": 0.0,
"solved": 0.0,
"grpo": 0.0,
"n": 24
},
{
"policy": "beam-low",
"x": 0.75,
"mean": 0.75,
"solved": 0.75,
"grpo": 0.5,
"n": 24
},
{
"policy": "beam-medium",
"x": 0.75,
"mean": 0.75,
"solved": 0.75,
"grpo": null,
"n": 8
},
{
"policy": "beam-high",
"x": 1.0,
"mean": 1.0,
"solved": 1.0,
"grpo": null,
"n": 8
}
],
"top_known": true
}
},
{
"suite": "rg-letter_counting",
"env": "reasoning_gym_env",
"notes": "reasoning_gym family letter_counting",
"episodes": 64,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-high": {
"episodes": 8,
"mean": 0.75,
"std": 0.4330127018922193,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 2997,
"invalid_steps": 0,
"steps": 8,
"pass@1": 0.75,
"pass@k": 0.75,
"k": 1
},
"beam-low": {
"episodes": 24,
"mean": 0.6666666666666666,
"std": 0.4714045207910317,
"within_task_std": 0.35355339059327373,
"grpo_signal": 0.75,
"out_tokens": 4284,
"invalid_steps": 0,
"steps": 24,
"pass@1": 0.6666666666666666,
"pass@k": 1.0,
"k": 3
},
"null": {
"episodes": 8,
"mean": 0.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 8,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 1
},
"small": {
"episodes": 24,
"mean": 0.04512210708291711,
"std": 0.1993864118894454,
"within_task_std": 0.06381229579951248,
"grpo_signal": 0.5,
"out_tokens": 425,
"invalid_steps": 0,
"steps": 24,
"pass@1": 0.041666666666666664,
"pass@k": 0.125,
"k": 3
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_solve_rate": 0.0,
"floor_policy": "null"
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-high",
"pass@1": 0.75,
"pass@k": 0.75,
"k": 1
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 0.75,
"ci95": [
0.375,
1.0
],
"p": 0.0,
"n": [
8,
8
],
"between_beam_efforts": {
"stronger": "beam-high",
"weaker": "beam-low",
"diff": 0.08333333333333337,
"ci95": [
-0.29166666666666663,
0.41666666666666663
],
"p": 0.3505,
"separates": false
}
},
"statistical.headroom": {
"pass": true,
"best_mean": 0.75,
"max": 1.0,
"room_fraction": 0.25
},
"trainee.reachability": {
"pass": true,
"pass@1": 0.041666666666666664,
"pass@k": 0.125,
"k": 3
},
"trainee.grpo_signal": {
"pass": true,
"share": 0.5,
"threshold": 0.2
},
"trainee.rule": {
"prediction": "improves"
},
"trainee.room_below_reference": {
"pass": true,
"reference_minus_trainee": 0.7048778929170829,
"p": 0.00025
},
"statistical.adversarial_floor": {
"pass": false,
"strategy": "constant",
"label": "always \"1\"",
"answer": "1",
"score": 0.35,
"learner_mean": 0.045,
"added_after_training": true
}
},
"verdict": "worth training the small model on",
"group": "worth",
"reference": "beam-high",
"trainee": "small",
"judged_for": "small model",
"tasks": 8,
"provisional": false,
"gameable": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": 0.0,
"solved": 0.0,
"grpo": null,
"n": 8
},
{
"policy": "small",
"x": 0.0451,
"mean": 0.04512210708291711,
"solved": 0.041666666666666664,
"grpo": 0.5,
"n": 24
},
{
"policy": "beam-low",
"x": 0.6667,
"mean": 0.6666666666666666,
"solved": 0.6666666666666666,
"grpo": 0.75,
"n": 24
},
{
"policy": "beam-high",
"x": 0.75,
"mean": 0.75,
"solved": 0.75,
"grpo": null,
"n": 8
}
],
"top_known": true
}
},
{
"suite": "rg-maze",
"env": "reasoning_gym_env",
"notes": "reasoning_gym family maze",
"episodes": 72,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-high": {
"episodes": 8,
"mean": 1.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 31333,
"invalid_steps": 0,
"steps": 8,
"pass@1": 1.0,
"pass@k": 1.0,
"k": 1
},
"beam-low": {
"episodes": 24,
"mean": 0.75,
"std": 0.4330127018922193,
"within_task_std": 0.11785113019775792,
"grpo_signal": 0.25,
"out_tokens": 29548,
"invalid_steps": 0,
"steps": 24,
"pass@1": 0.75,
"pass@k": 0.875,
"k": 3
},
"beam-medium": {
"episodes": 8,
"mean": 0.875,
"std": 0.33071891388307384,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 18204,
"invalid_steps": 0,
"steps": 8,
"pass@1": 0.875,
"pass@k": 0.875,
"k": 1
},
"null": {
"episodes": 8,
"mean": 0.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 8,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 1
},
"small": {
"episodes": 24,
"mean": 0.0008869368531945219,
"std": 0.003216665538319247,
"within_task_std": 0.0012543181267562077,
"grpo_signal": 0.25,
"out_tokens": 371,
"invalid_steps": 0,
"steps": 24,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 3
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_solve_rate": 0.0,
"floor_policy": "null"
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-high",
"pass@1": 1.0,
"pass@k": 1.0,
"k": 1
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 1.0,
"ci95": [
1.0,
1.0
],
"p": 0.0,
"n": [
8,
8
],
"between_beam_efforts": {
"stronger": "beam-high",
"weaker": "beam-low",
"diff": 0.25,
"ci95": [
0.08333333333333337,
0.41666666666666663
],
"p": 0.00175,
"separates": true
}
},
"statistical.headroom": {
"pass": false,
"best_mean": 1.0,
"max": 1.0,
"room_fraction": 0.0
},
"trainee.reachability": {
"pass": false,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 3
},
"trainee.grpo_signal": {
"pass": true,
"share": 0.25,
"threshold": 0.2
},
"trainee.rule": {
"prediction": "improves"
},
"trainee.room_below_reference": {
"pass": true,
"reference_minus_trainee": 0.9991130631468055,
"p": 0.0
},
"statistical.adversarial_floor": {
"pass": false,
"strategy": "constant",
"label": "always \"5\"",
"answer": "5",
"score": 0.315,
"learner_mean": 0.001,
"added_after_training": true
}
},
"verdict": "worth training the small model on",
"group": "worth",
"reference": "beam-high",
"trainee": "small",
"judged_for": "small model",
"tasks": 8,
"provisional": false,
"gameable": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": 0.0,
"solved": 0.0,
"grpo": null,
"n": 8
},
{
"policy": "small",
"x": 0.0009,
"mean": 0.0008869368531945219,
"solved": 0.0,
"grpo": 0.25,
"n": 24
},
{
"policy": "beam-low",
"x": 0.75,
"mean": 0.75,
"solved": 0.75,
"grpo": 0.25,
"n": 24
},
{
"policy": "beam-medium",
"x": 0.875,
"mean": 0.875,
"solved": 0.875,
"grpo": null,
"n": 8
},
{
"policy": "beam-high",
"x": 1.0,
"mean": 1.0,
"solved": 1.0,
"grpo": null,
"n": 8
}
],
"top_known": true
}
},
{
"suite": "rg-propositional_logic",
"env": "reasoning_gym_env",
"notes": "reasoning_gym family propositional_logic",
"episodes": 72,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-high": {
"episodes": 8,
"mean": 0.8125,
"std": 0.3247595264191645,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 18651,
"invalid_steps": 0,
"steps": 8,
"pass@1": 0.75,
"pass@k": 0.75,
"k": 1
},
"beam-low": {
"episodes": 24,
"mean": 0.6875,
"std": 0.369754986443726,
"within_task_std": 0.13258252147247768,
"grpo_signal": 0.375,
"out_tokens": 8470,
"invalid_steps": 0,
"steps": 24,
"pass@1": 0.5833333333333334,
"pass@k": 0.75,
"k": 3
},
"beam-medium": {
"episodes": 8,
"mean": 0.625,
"std": 0.375,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 5669,
"invalid_steps": 0,
"steps": 8,
"pass@1": 0.5,
"pass@k": 0.5,
"k": 1
},
"null": {
"episodes": 8,
"mean": 0.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 8,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 1
},
"small": {
"episodes": 24,
"mean": 0.010416666666666666,
"std": 0.020305821551685345,
"within_task_std": 0.011785113019775792,
"grpo_signal": 0.5,
"out_tokens": 512,
"invalid_steps": 0,
"steps": 24,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 3
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_solve_rate": 0.0,
"floor_policy": "null"
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-high",
"pass@1": 0.75,
"pass@k": 0.75,
"k": 1
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 0.8125,
"ci95": [
0.53125,
1.0
],
"p": 0.0,
"n": [
8,
8
],
"between_beam_efforts": {
"stronger": "beam-high",
"weaker": "beam-medium",
"diff": 0.1875,
"ci95": [
-0.1875,
0.46875
],
"p": 0.204,
"separates": false
}
},
"statistical.headroom": {
"pass": true,
"best_mean": 0.8125,
"max": 1.0,
"room_fraction": 0.1875
},
"trainee.reachability": {
"pass": false,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 3
},
"trainee.grpo_signal": {
"pass": true,
"share": 0.5,
"threshold": 0.2
},
"trainee.rule": {
"prediction": "improves"
},
"trainee.room_below_reference": {
"pass": true,
"reference_minus_trainee": 0.8020833333333334,
"p": 0.0
},
"statistical.adversarial_floor": {
"pass": false,
"strategy": "replay input",
"label": "the premises joined with 'and'",
"answer": "the premises joined with 'and'",
"score": 0.993,
"learner_mean": 0.01,
"added_after_training": true
}
},
"verdict": "worth training the small model on",
"group": "worth",
"reference": "beam-high",
"trainee": "small",
"judged_for": "small model",
"tasks": 8,
"provisional": false,
"gameable": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": 0.0,
"solved": 0.0,
"grpo": null,
"n": 8
},
{
"policy": "small",
"x": 0.0104,
"mean": 0.010416666666666666,
"solved": 0.0,
"grpo": 0.5,
"n": 24
},
{
"policy": "beam-low",
"x": 0.6875,
"mean": 0.6875,
"solved": 0.5833333333333334,
"grpo": 0.375,
"n": 24
},
{
"policy": "beam-medium",
"x": 0.625,
"mean": 0.625,
"solved": 0.5,
"grpo": null,
"n": 8
},
{
"policy": "beam-high",
"x": 0.8125,
"mean": 0.8125,
"solved": 0.75,
"grpo": null,
"n": 8
}
],
"top_known": true
}
},
{
"suite": "rg-sudoku",
"env": "reasoning_gym_env",
"notes": "reasoning_gym family sudoku",
"episodes": 72,
"errors": 1,
"harness_errors": 0,
"tiers": {
"beam-high": {
"episodes": 8,
"mean": 1.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 116723,
"invalid_steps": 0,
"steps": 8,
"pass@1": 1.0,
"pass@k": 1.0,
"k": 1
},
"beam-low": {
"episodes": 24,
"mean": 0.6660628476693008,
"std": 0.2533264824118278,
"within_task_std": 0.14614835638002605,
"grpo_signal": 1.0,
"out_tokens": 70465,
"invalid_steps": 0,
"steps": 24,
"pass@1": 0.16666666666666666,
"pass@k": 0.375,
"k": 3
},
"beam-medium": {
"episodes": 8,
"mean": 0.8328189300411523,
"std": 0.3335932120868177,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 75140,
"invalid_steps": 1,
"steps": 8,
"pass@1": 0.75,
"pass@k": 0.75,
"k": 1
},
"null": {
"episodes": 8,
"mean": 0.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 8,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 1
},
"small": {
"episodes": 23,
"mean": 0.17246376811594202,
"std": 0.056071954119988054,
"within_task_std": 0.01768579918330598,
"grpo_signal": 1.0,
"out_tokens": 1472,
"invalid_steps": 0,
"steps": 23,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 3
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_solve_rate": 0.0,
"floor_policy": "null"
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-high",
"pass@1": 1.0,
"pass@k": 1.0,
"k": 1
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 1.0,
"ci95": [
1.0,
1.0
],
"p": 0.0,
"n": [
8,
8
],
"between_beam_efforts": {
"stronger": "beam-high",
"weaker": "beam-low",
"diff": 0.3339371523306992,
"ci95": [
0.23736650761190647,
0.44154141104294486
],
"p": 0.0,
"separates": true
}
},
"statistical.headroom": {
"pass": false,
"best_mean": 1.0,
"max": 1.0,
"room_fraction": 0.0
},
"trainee.reachability": {
"pass": false,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 3
},
"trainee.grpo_signal": {
"pass": true,
"share": 1.0,
"threshold": 0.2
},
"trainee.rule": {
"prediction": "improves"
},
"trainee.room_below_reference": {
"pass": true,
"reference_minus_trainee": 0.827536231884058,
"p": 0.0
},
"statistical.adversarial_floor": {
"pass": true,
"strategy": "constant",
"label": "always one fixed answer",
"answer": "8 6 2 1 9 3 5 4 7\n4 5 3 8 2 7 1 6 9\n1...",
"score": 0.123,
"learner_mean": 0.172,
"added_after_training": true
}
},
"verdict": "worth training the small model on",
"group": "worth",
"reference": "beam-high",
"trainee": "small",
"judged_for": "small model",
"tasks": 8,
"provisional": false,
"gameable": false,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": 0.0,
"solved": 0.0,
"grpo": null,
"n": 8
},
{
"policy": "small",
"x": 0.1725,
"mean": 0.17246376811594202,
"solved": 0.0,
"grpo": 1.0,
"n": 23
},
{
"policy": "beam-low",
"x": 0.6661,
"mean": 0.6660628476693008,
"solved": 0.16666666666666666,
"grpo": 1.0,
"n": 24
},
{
"policy": "beam-medium",
"x": 0.8328,
"mean": 0.8328189300411523,
"solved": 0.75,
"grpo": null,
"n": 8
},
{
"policy": "beam-high",
"x": 1.0,
"mean": 1.0,
"solved": 1.0,
"grpo": null,
"n": 8
}
],
"top_known": true
}
},
{
"suite": "rg-word_sorting",
"env": "reasoning_gym_env",
"notes": "reasoning_gym family word_sorting",
"episodes": 64,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-high": {
"episodes": 8,
"mean": 0.9222222222222223,
"std": 0.14185717038670784,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 3119,
"invalid_steps": 0,
"steps": 8,
"pass@1": 0.75,
"pass@k": 0.75,
"k": 1
},
"beam-low": {
"episodes": 24,
"mean": 0.9546296296296296,
"std": 0.18036076862419545,
"within_task_std": 0.06416339310766819,
"grpo_signal": 0.25,
"out_tokens": 5708,
"invalid_steps": 0,
"steps": 24,
"pass@1": 0.9166666666666666,
"pass@k": 1.0,
"k": 3
},
"null": {
"episodes": 8,
"mean": 0.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 8,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 1
},
"small": {
"episodes": 24,
"mean": 0.20011574074074076,
"std": 0.20387904779753585,
"within_task_std": 0.0918835877736538,
"grpo_signal": 0.875,
"out_tokens": 451,
"invalid_steps": 0,
"steps": 24,
"pass@1": 0.041666666666666664,
"pass@k": 0.125,
"k": 3
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_solve_rate": 0.0,
"floor_policy": "null"
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-low",
"pass@1": 0.9166666666666666,
"pass@k": 1.0,
"k": 3
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 0.9546296296296296,
"ci95": [
0.8722222222222221,
1.0
],
"p": 0.0,
"n": [
24,
8
],
"between_beam_efforts": {
"stronger": "beam-low",
"weaker": "beam-high",
"diff": 0.03240740740740733,
"ci95": [
-0.09074074074074068,
0.1611111111111112
],
"p": 0.3065,
"separates": false
}
},
"statistical.headroom": {
"pass": false,
"best_mean": 0.9546296296296296,
"max": 1.0,
"room_fraction": 0.045370370370370394
},
"statistical.variance_bound": {
"pass": true,
"within_task_std": 0.06416339310766819,
"signal": 0.9546296296296296
},
"trainee.reachability": {
"pass": true,
"pass@1": 0.041666666666666664,
"pass@k": 0.125,
"k": 3
},
"trainee.grpo_signal": {
"pass": true,
"share": 0.875,
"threshold": 0.2
},
"trainee.rule": {
"prediction": "improves"
},
"trainee.room_below_reference": {
"pass": true,
"reference_minus_trainee": 0.7545138888888888,
"p": 0.0
},
"statistical.adversarial_floor": {
"pass": false,
"strategy": "replay input",
"label": "the input words, unsorted",
"answer": "the input words, unsorted",
"score": 0.256,
"learner_mean": 0.2,
"added_after_training": true
}
},
"verdict": "worth training the small model on",
"group": "worth",
"reference": "beam-low",
"trainee": "small",
"judged_for": "small model",
"tasks": 8,
"provisional": false,
"gameable": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": 0.0,
"solved": 0.0,
"grpo": null,
"n": 8
},
{
"policy": "small",
"x": 0.2001,
"mean": 0.20011574074074076,
"solved": 0.041666666666666664,
"grpo": 0.875,
"n": 24
},
{
"policy": "beam-low",
"x": 0.9546,
"mean": 0.9546296296296296,
"solved": 0.9166666666666666,
"grpo": 0.25,
"n": 24
},
{
"policy": "beam-high",
"x": 0.9222,
"mean": 0.9222222222222223,
"solved": 0.75,
"grpo": null,
"n": 8
}
],
"top_known": true
}
},
{
"suite": "snake_env",
"env": "snake_env",
"notes": "snake; capped at 20 moves",
"episodes": 8,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": -0.15100000000000033,
"std": 0.6924524051032912,
"within_task_std": 0.6495158657126944,
"grpo_signal": 1.0,
"out_tokens": 39206,
"invalid_steps": 0,
"steps": 96
},
"null": {
"episodes": 2,
"mean": -0.9895,
"std": 0.003500000000000003,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 23
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_mean": -0.9895,
"top": 1.019999999999998,
"gap_fraction": 0.9957879088206146
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-low",
"gain_over_floor": 0.8384999999999997,
"ci95": [
0.33233333333333326,
1.3433333333333324
],
"p": 0.002
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 0.8384999999999997,
"ci95": [
0.3311624999999997,
1.3449999999999989
],
"p": 0.00175,
"n": [
6,
2
]
},
"statistical.variance_bound": {
"pass": true,
"within_task_std": 0.6495158657126944,
"signal": 0.8384999999999997
},
"statistical.grpo_signal": {
"pass": true,
"reference": "beam-low",
"share": 1.0,
"threshold": 0.2
}
},
"verdict": "learnable for Beam; not yet tested with a small model",
"group": "worth",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 2,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0042,
"mean": -0.9895,
"solved": null,
"grpo": null,
"n": 2
},
{
"policy": "beam-low",
"x": 0.4197,
"mean": -0.15100000000000033,
"solved": null,
"grpo": 1.0,
"n": 6
}
],
"top_known": false
}
},
{
"suite": "sophistry_bench_sprint_env",
"env": "sophistry_bench_sprint_env",
"notes": "single argued answer with verbatim citations",
"episodes": 16,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 12,
"mean": 0.9015624999999999,
"std": 0.1377631241857438,
"within_task_std": 0.08101206371139363,
"grpo_signal": 0.75,
"out_tokens": 7590,
"invalid_steps": 0,
"steps": 12
},
"null": {
"episodes": 4,
"mean": 0.5,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 4
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_mean": 0.5,
"top": 1.0,
"gap_fraction": 1.0
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-low",
"gain_over_floor": 0.40156249999999993,
"ci95": [
0.3151041666666665,
0.46875
],
"p": 0.0
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 0.40156249999999993,
"ci95": [
0.3135416666666666,
0.4671875
],
"p": 0.0,
"n": [
12,
4
]
},
"statistical.variance_bound": {
"pass": true,
"within_task_std": 0.08101206371139363,
"signal": 0.40156249999999993
},
"statistical.grpo_signal": {
"pass": true,
"reference": "beam-low",
"share": 0.75,
"threshold": 0.2
}
},
"verdict": "learnable for Beam; not yet tested with a small model",
"group": "worth",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 4,
"provisional": false,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0,
"mean": 0.5,
"solved": null,
"grpo": null,
"n": 4
},
{
"policy": "beam-low",
"x": 0.8031,
"mean": 0.9015624999999999,
"solved": null,
"grpo": 0.75,
"n": 12
}
],
"top_known": false
}
},
{
"suite": "wildfire_env",
"env": "wildfire_env",
"notes": "128-step episode capped at 15, scored per step",
"episodes": 8,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": -0.35797048611111104,
"std": 0.5453663550613294,
"within_task_std": 0.25352614683863084,
"grpo_signal": 1.0,
"out_tokens": 51941,
"invalid_steps": 0,
"steps": 69
},
"null": {
"episodes": 2,
"mean": -1.1283333333333334,
"std": 0.07499999999999996,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 30
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_mean": -1.1283333333333334,
"top": 0.48908854166666665,
"gap_fraction": 0.9556848081983105
},
"statistical.reward_reachability": {
"pass": true,
"reference": "beam-low",
"gain_over_floor": 0.7703628472222224,
"ci95": [
0.33333333333333337,
1.2290312500000005
],
"p": 0.0
},
"statistical.difficulty_separation": {
"pass": true,
"reference_minus_floor": 0.7703628472222224,
"ci95": [
0.35955555555555574,
1.2428368055555556
],
"p": 0.0,
"n": [
6,
2
]
},
"statistical.variance_bound": {
"pass": true,
"within_task_std": 0.25352614683863084,
"signal": 0.7703628472222224
},
"statistical.grpo_signal": {
"pass": true,
"reference": "beam-low",
"share": 1.0,
"threshold": 0.2
}
},
"verdict": "learnable for Beam; not yet tested with a small model",
"group": "worth",
"reference": "beam-low",
"trainee": null,
"judged_for": "Beam",
"tasks": 2,
"provisional": true,
"rail": {
"rungs": [
{
"policy": "null",
"x": 0.0443,
"mean": -1.1283333333333334,
"solved": null,
"grpo": null,
"n": 2
},
{
"policy": "beam-low",
"x": 0.4995,
"mean": -0.35797048611111104,
"solved": null,
"grpo": 1.0,
"n": 6
}
],
"top_known": false
}
}
],
"connect4": {
"env": "connect4_env",
"as_shipped": {
"null": {
"won": 3,
"games": 3
},
"random": {
"won": 9,
"games": 9
},
"beam-low": {
"won": 9,
"games": 9
}
},
"with_beams_fix": {
"null": {
"won": 0,
"games": 3
},
"random": {
"won": 1,
"games": 9
},
"beam-low": {
"won": 8,
"games": 9
}
},
"diagnosis": "DIAGNOSIS:\nThe environment never makes an opponent move, so the agent effectively plays both sides, and `_check_win_or_draw` returns `+1.0` for any four-in-a-row regardless of which player made it. Every finished game therefore ends in a win and pays the top reward. The fix is to make the agent player 1, have a built-in opponent respond after each legal agent move, and score a win as `+1` only when player 1 wins and `-1` when the opponent wins.\n\nFIX:\nUpdated `server/connect4_environment.py` so each `step` places the agent's piece as player 1, then automatically plays an opponent move (immediate win/block/center preference) unless the game is already over. Reward polarity now depends on the winning player, and invalid moves still end the episode with `-1`."
},
"dipg": {
"default_markers": {
"cleared_format_gate": 0,
"answers": 12,
"fully_correct": 0,
"mean_score": -10.0
},
"square_brackets": {
"cleared_format_gate": 7,
"answers": 12,
"fully_correct": 0,
"mean_score": -4.17
},
"pilot": {
"dipg_safety_env-brackets": {
"cleared_format_gate": 6,
"answers": 12,
"fully_correct": 0,
"mean_score": -5.0
},
"dipg_safety_env-plain": {
"cleared_format_gate": 0,
"answers": 12,
"fully_correct": 0,
"mean_score": -10.0
},
"dipg_safety_env": {
"cleared_format_gate": 0,
"answers": 12,
"fully_correct": 0,
"mean_score": -10.0
}
}
},
"effort": [
{
"suite": "maze_env",
"rungs": {
"beam-low": {
"solved": 1.0,
"mean": 9.283,
"episodes": 9
},
"beam-high": {
"solved": 1.0,
"mean": 9.35,
"episodes": 3
}
}
},
{
"suite": "os-tic_tac_toe",
"rungs": {
"beam-low": {
"solved": 0.8888888888888888,
"mean": 0.889,
"episodes": 9
},
"beam-high": {
"solved": 1.0,
"mean": 1.0,
"episodes": 3
}
}
},
{
"suite": "rg-basic_arithmetic",
"rungs": {
"beam-low": {
"solved": 0.875,
"mean": 0.875,
"episodes": 24
},
"beam-high": {
"solved": 0.875,
"mean": 0.875,
"episodes": 8
}
}
},
{
"suite": "rg-chain_sum",
"rungs": {
"beam-low": {
"solved": 1.0,
"mean": 1.0,
"episodes": 24
},
"beam-high": {
"solved": 1.0,
"mean": 1.0,
"episodes": 8
}
}
},
{
"suite": "rg-countdown",
"rungs": {
"beam-low": {
"solved": 0.6666666666666666,
"mean": 0.683,
"episodes": 24
},
"beam-medium": {
"solved": 0.75,
"mean": 0.762,
"episodes": 8
},
"beam-high": {
"solved": 1.0,
"mean": 1.0,
"episodes": 8
}
}
},
{
"suite": "rg-game_of_life",
"rungs": {
"beam-low": {
"solved": 0.375,
"mean": 0.978,
"episodes": 24
},
"beam-medium": {
"solved": 0.875,
"mean": 0.999,
"episodes": 8
},
"beam-high": {
"solved": 0.875,
"mean": 0.996,
"episodes": 8
},
"beam-max": {
"solved": 0.875,
"mean": 0.998,
"episodes": 8
}
}
},
{
"suite": "rg-leg_counting",
"rungs": {
"beam-low": {
"solved": 0.75,
"mean": 0.75,
"episodes": 24
},
"beam-medium": {
"solved": 0.75,
"mean": 0.75,
"episodes": 8
},
"beam-high": {
"solved": 1.0,
"mean": 1.0,
"episodes": 8
}
}
},
{
"suite": "rg-letter_counting",
"rungs": {
"beam-low": {
"solved": 0.6666666666666666,
"mean": 0.667,
"episodes": 24
},
"beam-high": {
"solved": 0.75,
"mean": 0.75,
"episodes": 8
}
}
},
{
"suite": "rg-maze",
"rungs": {
"beam-low": {
"solved": 0.75,
"mean": 0.75,
"episodes": 24
},
"beam-medium": {
"solved": 0.875,
"mean": 0.875,
"episodes": 8
},
"beam-high": {
"solved": 1.0,
"mean": 1.0,
"episodes": 8
}
}
},
{
"suite": "rg-propositional_logic",
"rungs": {
"beam-low": {
"solved": 0.5833333333333334,
"mean": 0.688,
"episodes": 24
},
"beam-medium": {
"solved": 0.5,
"mean": 0.625,
"episodes": 8
},
"beam-high": {
"solved": 0.75,
"mean": 0.812,
"episodes": 8
}
}
},
{
"suite": "rg-sudoku",
"rungs": {
"beam-low": {
"solved": 0.16666666666666666,
"mean": 0.666,
"episodes": 24
},
"beam-medium": {
"solved": 0.75,
"mean": 0.833,
"episodes": 8
},
"beam-high": {
"solved": 1.0,
"mean": 1.0,
"episodes": 8
}
}
},
{
"suite": "rg-word_sorting",
"rungs": {
"beam-low": {
"solved": 0.9166666666666666,
"mean": 0.955,
"episodes": 24
},
"beam-high": {
"solved": 0.75,
"mean": 0.922,
"episodes": 8
}
}
}
],
"effort_summary": {
"suites": 12,
"compared": 12,
"higher_solved_more": 8,
"same": 3,
"higher_solved_less": 1
},
"census": {
"spaces": 269,
"groups": [
{
"group": "running, but no standard OpenEnv API",
"count": 79,
"share": 0.294
},
{
"group": "runs and passes the floor checks",
"count": 63,
"share": 0.234
},
{
"group": "asleep, no answer within 10 minutes",
"count": 36,
"share": 0.134
},
{
"group": "scores above zero for doing nothing",
"count": 30,
"share": 0.112
},
{
"group": "broken (build or runtime error on the Hub)",
"count": 25,
"share": 0.093
},
{
"group": "crashes on reset or step",
"count": 17,
"share": 0.063
},
{
"group": "refuses a session",
"count": 10,
"share": 0.037
},
{
"group": "paused by its owner",
"count": 9,
"share": 0.033
}
],
"sum_of_shares": 1.0,
"do_nothing_highest": [
{
"space": "netra412005/flash-sale-env-v2",
"do_nothing_score": 45275.3013708072
},
{
"space": "tuhindev2029/cacheforge",
"do_nothing_score": 12.7405
},
{
"space": "stavust28/hospitalmanage_triage_env",
"do_nothing_score": 7.000000000000001
},
{
"space": "Jishnu-Vijayan-03/disaster-response",
"do_nothing_score": 6.859200000000001
},
{
"space": "Ev3Dev/Bio-EnvRL",
"do_nothing_score": 5.977099354575067
}
],
"do_nothing_at_least_1": 13,
"second_pass_done": 156
},
"training": {
"pre_registration": {
"sha256": "d11a5928bce2f1b045b593e699370612779b6b45950cbdbee256ef6a23ccc776",
"committed_utc": "2026-10-30 16:23:45",
"first_checkpoint_utc": "2026-10-30 16:25"
},
"held": 6,
"families": 6,
"rows": [
{
"family": "basic_arithmetic",
"prediction": "improves",
"held": true,
"reward_before": 0.052,
"reward_after": 0.483,
"p": 0.0,
"accuracy_before": 0.01,
"accuracy_after": 0.48,
"lenient_accuracy_before": 0.255,
"lenient_reward_before": 0.268,
"baseline": {
"strategy": "always answer 0",
"score": 0.02
},
"learned": "skill and format",
"learned_text": "Partly skill, partly format: read leniently, the untrained model was already right on 25.5%.",
"beam_audit": "learned_format",
"beam_explanation": "The before answers often contained the correct numeric result inside longer expressions or explanations, so the grader only gave partial credit through the substring branch. After training, the model emits the bare value, which the grader scores as an exact match. This indicates the gain is mainly from answer formatting rather than improved arithmetic."
},
{
"family": "chain_sum",
"prediction": "improves",
"held": true,
"reward_before": 0.038,
"reward_after": 0.3,
"p": 0.0,
"accuracy_before": 0.005,
"accuracy_after": 0.23,
"lenient_accuracy_before": 0.4,
"lenient_reward_before": 0.4,
"baseline": {
"strategy": "always answer 0",
"score": 0.02
},
"learned": "mostly format",
"learned_text": "Mostly format: read leniently, the untrained model was already right on 40%.",
"beam_audit": "learned_format",
"beam_explanation": "The model improved mainly by emitting concise answers or calculation chains instead of verbose explanations, which raises string-similarity reward against the oracle answer. The same numeric answer can score very differently depending on surrounding text, as shown by the verbose versus concise versions of 757. Thus the gain is largely presentation-driven rather than evidence of better arithmetic."
},
{
"family": "letter_counting",
"prediction": "improves",
"held": true,
"reward_before": 0.137,
"reward_after": 0.35,
"p": 0.0,
"accuracy_before": 0.135,
"accuracy_after": 0.35,
"lenient_accuracy_before": null,
"lenient_reward_before": null,
"baseline": {
"strategy": "always answer 1",
"score": 0.35
},
"learned": "gamed",
"learned_text": "It answers \"1\" to everything; that alone scores 0.350.",
"beam_audit": "gamed_grader",
"beam_explanation": "After training the model outputs the constant answer \"1\" on every shown task, so it only earns reward when the oracle count happens to be 1. This is not solving the counting task; it exploits the grader's exact-match reward for a frequent answer without depending on the question."
},
{
"family": "propositional_logic",
"prediction": "improves",
"held": true,
"reward_before": 0.001,
"reward_after": 0.993,
"p": 0.0,
"accuracy_before": 0.0,
"accuracy_after": 0.99,
"lenient_accuracy_before": null,
"lenient_reward_before": null,
"baseline": {
"strategy": "join the premises with 'and'",
"score": 0.993
},
"learned": "gamed",
"learned_text": "It joins the premises with \"and\"; that alone scores 0.993.",
"beam_audit": "gamed_grader",
"beam_explanation": "The trained model does not appear to infer new conclusions; it emits conjunctions of premises and tautological variants, which are trivially entailed and receive full score. The pre-training answers often contained a plausible conclusion but were wrapped in prose, so much of the apparent gain comes from bypassing the parser and exploiting the grader's acceptance of premise conjunctions."
},
{
"family": "word_sorting",
"prediction": "improves",
"held": true,
"reward_before": 0.221,
"reward_after": 0.253,
"p": 0.0076,
"accuracy_before": 0.01,
"accuracy_after": 0.02,
"lenient_accuracy_before": null,
"lenient_reward_before": null,
"baseline": {
"strategy": "repeat the input words unsorted",
"score": 0.256
},
"learned": "no real gain",
"learned_text": "Repeating the input words unsorted already scores 0.256.",
"beam_audit": "gamed_grader",
"beam_explanation": "The grader awards at least 0.2 whenever the answer contains the same words as the oracle, so repeating the input words can earn reward without actually sorting. Held-out scores remain near that 0.2 floor (0.221 to 0.253) and exact accuracy stays around 1-2%, while many shown answers are just input-order permutations scoring 0.2. This points to exploiting the grader loophole rather than learning sorting."
},
{
"family": "leg_counting",
"prediction": "no change",
"held": true,
"reward_before": 0.0,
"reward_after": 0.0,
"p": 0.6534,
"accuracy_before": 0.0,
"accuracy_after": 0.0,
"lenient_accuracy_before": null,
"lenient_reward_before": null,
"baseline": null,
"learned": "no change",
"learned_text": "No reward on any attempt, so nothing to learn from; this prediction could not fail.",
"beam_audit": "no_change",
"beam_explanation": "The held-out mean score stayed 0.000 before and after training, and the sampled answers remain incomplete or use incorrect leg counts. Training changed phrasing and often assigns six legs to every animal, but it did not produce answers that match the oracle or earn grader reward."
}
]
},
"forward_test": {
"family": "maze",
"written_at": "2026-10-31 22:31:12 UTC",
"claims": [
{
"id": "P1",
"claim": "Reward rises.",
"test": "Mean held-out reward after training is above before, paired bootstrap p <= 0.05 (the same test as the six-family run)."
},
{
"id": "P2",
"claim": "The model collapses onto one constant answer.",
"test": "After training, a single answer string makes up at least 50% of the 200 greedy held-out answers."
},
{
"id": "P3",
"claim": "The constant is 5 or 6.",
"test": "The most common answer after training is '5' or '6', the two most common correct answers on the training tasks."
},
{
"id": "P4",
"claim": "No maze-solving skill is learned.",
"test": "On held-out tasks whose correct answer differs from the model's most common answer after training, accuracy does not rise: paired bootstrap p > 0.05 for a gain."
}
],
"evidence": {
"shortlist_verdict": "worth training the small model on",
"small_model_mean_reward_on_ladder": 0.001,
"small_model_grpo_signal_on_ladder": 0.25,
"gaming_check": {
"constant_answer": "5",
"its_held_out_score": 0.315,
"most_common_correct_answers_on_training_tasks": {
"5": 114,
"6": 112,
"7": 66,
"8": 52
},
"training_tasks": 400
}
},
"sha256": "4cca90e80a446fcb9286c2e9007450c7fd31e2d692f32b24d256eca118477cc8",
"result": {
"collapsed_answer_start": "We are given a 5x5 grid:",
"collapsed_answer_numbers": "1 to 20",
"collapsed_answer_chars": 82,
"collapsed_share": 0.98,
"held_out_answers_inside_it": 200,
"held_out_tasks": 200,
"its_mean_score_if_given_everywhere": 0.013,
"constant_5_score": 0.315,
"reward_before": 0.004,
"reward_after": 0.013,
"exact_before": 0,
"exact_after": 0,
"bare_numbers_before": 20,
"minutes": 26.4,
"claims": [
{
"id": "P1",
"held": true,
"claim": "Reward rises.",
"test": "Mean held-out reward after training is above before, paired bootstrap p <= 0.05 (the same test as the six-family run).",
"measured": "mean reward 0.004 \u2192 0.013, paired p < 0.001"
},
{
"id": "P2",
"held": true,
"claim": "The model collapses onto one constant answer.",
"test": "After training, a single answer string makes up at least 50% of the 200 greedy held-out answers.",
"measured": "196 of 200 answers are the same text"
},
{
"id": "P3",
"held": false,
"claim": "The constant is 5 or 6.",
"test": "The most common answer after training is '5' or '6', the two most common correct answers on the training tasks.",
"measured": "the constant is not a number: a grid of the numbers 1 to 20, cut off at 64 tokens"
},
{
"id": "P4",
"held": true,
"claim": "No maze-solving skill is learned.",
"test": "On held-out tasks whose correct answer differs from the model's most common answer after training, accuracy does not rise: paired bootstrap p > 0.05 for a gain.",
"measured": "exact answers 0 \u2192 0 of 200"
}
]
}
},
"pelicans": [
{
"svg_b64": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA2MDAgNDAwIiB3aWR0aD0iNjAwIiBoZWlnaHQ9IjQwMCI+CiAgPHJlY3Qgd2lkdGg9IjYwMCIgaGVpZ2h0PSI0MDAiIGZpbGw9IiNiZmVhZmYiLz4KICA8ZyBpZD0iYmljeWNsZSI+CiAgICA8Y2lyY2xlIGN4PSIxODAiIGN5PSIyODAiIHI9IjYwIiBmaWxsPSJub25lIiBzdHJva2U9IiMyMjIiIHN0cm9rZS13aWR0aD0iOCIvPgogICAgPGNpcmNsZSBjeD0iMTgwIiBjeT0iMjgwIiByPSI4IiBmaWxsPSIjMjIyIi8+CiAgICA8Y2lyY2xlIGN4PSI0MjAiIGN5PSIyODAiIHI9IjYwIiBmaWxsPSJub25lIiBzdHJva2U9IiMyMjIiIHN0cm9rZS13aWR0aD0iOCIvPgogICAgPGNpcmNsZSBjeD0iNDIwIiBjeT0iMjgwIiByPSI4IiBmaWxsPSIjMjIyIi8+CiAgICA8bGluZSB4MT0iMTgwIiB5MT0iMjgwIiB4Mj0iMzAwIiB5Mj0iMTkwIiBzdHJva2U9IiMzMzMiIHN0cm9rZS13aWR0aD0iNyIvPgogICAgPGxpbmUgeDE9IjMwMCIgeTE9IjE5MCIgeDI9IjQyMCIgeTI9IjI4MCIgc3Ryb2tlPSIjMzMzIiBzdHJva2Utd2lkdGg9IjciLz4KICAgIDxsaW5lIHgxPSIzMDAiIHkxPSIxOTAiIHgyPSIzMDAiIHkyPSIyNTAiIHN0cm9rZT0iIzMzMyIgc3Ryb2tlLXdpZHRoPSI3Ii8+CiAgICA8bGluZSB4MT0iMjgwIiB5MT0iMjUwIiB4Mj0iMzQwIiB5Mj0iMjUwIiBzdHJva2U9IiMzMzMiIHN0cm9rZS13aWR0aD0iNyIvPgogICAgPGxpbmUgeDE9IjQyMCIgeTE9IjI4MCIgeDI9IjQyMCIgeTI9IjIzMCIgc3Ryb2tlPSIjMzMzIiBzdHJva2Utd2lkdGg9IjciLz4KICAgIDxsaW5lIHgxPSI0MDAiIHkxPSIyMzAiIHgyPSI0NjAiIHkyPSIyMzAiIHN0cm9rZT0iIzMzMyIgc3Ryb2tlLXdpZHRoPSI3Ii8+CiAgPC9nPgogIDxnIGlkPSJwZWxpY2FuIj4KICAgIDxlbGxpcHNlIGN4PSIzMTAiIGN5PSIxNjUiIHJ4PSI2OCIgcnk9IjQ1IiBmaWxsPSIjZjJmMmYyIiBzdHJva2U9IiM1NTUiIHN0cm9rZS13aWR0aD0iMyIvPgogICAgPGVsbGlwc2UgY3g9IjI4MCIgY3k9IjE1NSIgcng9IjE4IiByeT0iMTYiIGZpbGw9IiNmMmYyZjIiIHN0cm9rZT0iIzU1NSIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgICA8Y2lyY2xlIGN4PSIyODgiIGN5PSIxNTAiIHI9IjQiIGZpbGw9IiMxMTEiLz4KICAgIDxwb2x5Z29uIHBvaW50cz0iMzAwLDE1OCA1MDAsMTUwIDUwMCwxNzIgMzE1LDE3OCIgZmlsbD0iI2Y3ZDc3NCIgc3Ryb2tlPSIjN2E2NDIwIiBzdHJva2Utd2lkdGg9IjIiLz4KICAgIDxlbGxpcHNlIGN4PSIzNjUiIGN5PSIxODIiIHJ4PSI0OCIgcnk9IjI0IiBmaWxsPSIjZThkMmE2IiBzdHJva2U9IiM3YTY0MjAiIHN0cm9rZS13aWR0aD0iMiIvPgogICAgPGVsbGlwc2UgY3g9IjMwNSIgY3k9IjE4NSIgcng9IjE0IiByeT0iOCIgZmlsbD0iI2U4ZDJhNiIgc3Ryb2tlPSIjN2E2NDIwIiBzdHJva2Utd2lkdGg9IjEuNSIvPgogIDwvZz4KPC9zdmc+",
"reward": 1.0,
"structure": 1.0,
"judged": false,
"policy": "beam-low"
},
{
"svg_b64": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyMjAiIGhlaWdodD0iMTUwIiB2aWV3Qm94PSIwIDAgMjIwIDE1MCI+CiAgPGcgc3Ryb2tlPSIjMTExIiBzdHJva2Utd2lkdGg9IjIiIGZpbGw9Im5vbmUiPgogICAgPGNpcmNsZSBjeD0iNjIiIGN5PSIxMTgiIHI9IjI1IiBmaWxsPSIjZThlOGU4Ii8+CiAgICA8Y2lyY2xlIGN4PSI2MiIgY3k9IjExOCIgcj0iNSIgZmlsbD0iIzMzMyIvPgogICAgPGNpcmNsZSBjeD0iMTU1IiBjeT0iMTE4IiByPSIyNSIgZmlsbD0iI2U4ZThlOCIvPgogICAgPGNpcmNsZSBjeD0iMTU1IiBjeT0iMTE4IiByPSI1IiBmaWxsPSIjMzMzIi8+CiAgICA8bGluZSB4MT0iNjIiIHkxPSIxMTgiIHgyPSI5MCIgeTI9Ijg1Ii8+CiAgICA8bGluZSB4MT0iOTAiIHkxPSI4NSIgeDI9IjEzMiIgeTI9Ijg1Ii8+CiAgICA8bGluZSB4MT0iMTMyIiB5MT0iODUiIHgyPSIxNTUiIHkyPSIxMTgiLz4KICAgIDxsaW5lIHgxPSI5MCIgeTE9Ijg1IiB4Mj0iNjIiIHkyPSI4NSIvPgogICAgPGxpbmUgeDE9IjYyIiB5MT0iODUiIHgyPSI2MiIgeTI9IjEwMiIvPgogICAgPGxpbmUgeDE9IjkwIiB5MT0iODUiIHgyPSI5MCIgeTI9IjcwIi8+CiAgICA8bGluZSB4MT0iMTMyIiB5MT0iODUiIHgyPSIxNDgiIHkyPSI3MiIvPgogICAgPGxpbmUgeDE9IjE0OCIgeTE9IjcyIiB4Mj0iMTYyIiB5Mj0iNzIiLz4KICA8L2c+CiAgPGcgZmlsbD0iIzcwOTBiOCIgc3Ryb2tlPSIjMjIyIiBzdHJva2Utd2lkdGg9IjIiPgogICAgPGVsbGlwc2UgY3g9Ijk1IiBjeT0iNjIiIHJ4PSIzNCIgcnk9IjE4Ii8+CiAgICA8Y2lyY2xlIGN4PSIxMjgiIGN5PSI0MiIgcj0iMTQiLz4KICA8L2c+CiAgPHBhdGggZD0iTTEzOSA0MCBDMTY1IDM4LCAxODggNDIsIDE5NiA0NSBMMTk4IDUwIEMxODYgNTYsIDE2NSA1OCwgMTQ0IDUyIFoiIGZpbGw9IiNmM2QxN2YiIHN0cm9rZT0iIzIyMiIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgPHBhdGggZD0iTTE0NCA1MiBDMTYwIDU4LCAxODAgNjAsIDE5MiA1OCBMMTkwIDY2IEMxNzQgNzAsIDE1NCA2NywgMTQwIDU4IFoiIGZpbGw9IiNlN2I4NmUiIHN0cm9rZT0iIzIyMiIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgPHBhdGggZD0iTTc2IDU0IEM2MCA0OCwgNTIgNjIsIDY2IDc0IEM4MiA3OCwgOTAgNjYsIDc2IDU0IFoiIGZpbGw9IiM4N2E4Y2UiIHN0cm9rZT0iIzIyMiIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgPGxpbmUgeDE9IjEyOCIgeTE9IjU2IiB4Mj0iMTA2IiB5Mj0iODIiIHN0cm9rZT0iIzIyMiIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgPGxpbmUgeDE9IjExMiIgeTE9IjYwIiB4Mj0iOTgiIHkyPSI4NSIgc3Ryb2tlPSIjMjIyIiBzdHJva2Utd2lkdGg9IjIiLz4KICA8ZWxsaXBzZSBjeD0iOTUiIGN5PSI3MiIgcng9IjgiIHJ5PSI1IiBmaWxsPSIjNGI2IiBzdHJva2U9IiMyMjIiIHN0cm9rZS13aWR0aD0iMiIvPgo8L3N2Zz4=",
"reward": 1.0,
"structure": 1.0,
"judged": false,
"policy": "beam-low"
},
{
"svg_b64": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MDAgMjYwIiB3aWR0aD0iNDAwIiBoZWlnaHQ9IjI2MCI+CiAgPGcgaWQ9ImJpY3ljbGUiPgogICAgPGNpcmNsZSBjeD0iMTEwIiBjeT0iMjAwIiByPSI0MCIgZmlsbD0ibm9uZSIgc3Ryb2tlPSIjMjIyIiBzdHJva2Utd2lkdGg9IjQiLz4KICAgIDxjaXJjbGUgY3g9IjI1MCIgY3k9IjIwMCIgcj0iNDAiIGZpbGw9Im5vbmUiIHN0cm9rZT0iIzIyMiIgc3Ryb2tlLXdpZHRoPSI0Ii8+CiAgICA8bGluZSB4MT0iMTEwIiB5MT0iMjAwIiB4Mj0iMTgwIiB5Mj0iMTMwIiBzdHJva2U9IiMyMjIiIHN0cm9rZS13aWR0aD0iNCIvPgogICAgPGxpbmUgeDE9IjE4MCIgeTE9IjEzMCIgeDI9IjI1MCIgeTI9IjIwMCIgc3Ryb2tlPSIjMjIyIiBzdHJva2Utd2lkdGg9IjQiLz4KICAgIDxsaW5lIHgxPSIxODAiIHkxPSIxMzAiIHgyPSIxODAiIHkyPSIxNjUiIHN0cm9rZT0iIzIyMiIgc3Ryb2tlLXdpZHRoPSI0Ii8+CiAgICA8bGluZSB4MT0iMTgwIiB5MT0iMTY1IiB4Mj0iMTUwIiB5Mj0iMTcwIiBzdHJva2U9IiMyMjIiIHN0cm9rZS13aWR0aD0iNCIvPgogICAgPGxpbmUgeDE9IjI1MCIgeTE9IjE0MCIgeDI9IjI5MCIgeTI9IjExMCIgc3Ryb2tlPSIjMjIyIiBzdHJva2Utd2lkdGg9IjQiLz4KICAgIDxsaW5lIHgxPSIyOTAiIHkxPSIxMTAiIHgyPSIyNzUiIHkyPSIxMDUiIHN0cm9rZT0iIzIyMiIgc3Ryb2tlLXdpZHRoPSI0Ii8+CiAgICA8cmVjdCB4PSIxNjUiIHk9IjEyMCIgd2lkdGg9IjM1IiBoZWlnaHQ9IjgiIHJ4PSI0IiBmaWxsPSIjNTUzMzExIi8+CiAgPC9nPgogIDxnIGlkPSJwZWxpY2FuIj4KICAgIDxlbGxpcHNlIGN4PSIxNzAiIGN5PSIxMjUiIHJ4PSI1NSIgcnk9IjM0IiBmaWxsPSIjZjBlNmQyIiBzdHJva2U9IiM2YjVhNDUiIHN0cm9rZS13aWR0aD0iMiIvPgogICAgPGNpcmNsZSBjeD0iMjI1IiBjeT0iOTgiIHI9IjI4IiBmaWxsPSIjZjBlNmQyIiBzdHJva2U9IiM2YjVhNDUiIHN0cm9rZS13aWR0aD0iMiIvPgogICAgPHBhdGggZD0iTTI0Miw5OCBMMzE4LDk0IEwyNDgsMTE2IFoiIGZpbGw9IiNmN2QzNTgiIHN0cm9rZT0iIzhhNmQyMiIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgICA8cGF0aCBkPSJNMjQ4LDExNCBRMjkwLDE0NSAyNDAsMTMyIFoiIGZpbGw9IiNmNWE5YjgiIHN0cm9rZT0iIzhhNmQyMiIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgICA8cGF0aCBkPSJNMTM1LDEwNSBRMTc1LDg1IDIxNSwxMDUgUTE3NSwxMjUgMTM1LDEwNSBaIiBmaWxsPSIjZDhjY2IwIiBzdHJva2U9IiM2YjVhNDUiIHN0cm9rZS13aWR0aD0iMiIvPgogICAgPGxpbmUgeDE9IjE2NSIgeTE9IjE1NSIgeDI9IjE3NSIgeTI9IjE4NSIgc3Ryb2tlPSIjNmI1YTQ1IiBzdHJva2Utd2lkdGg9IjMiLz4KICAgIDxsaW5lIHgxPSIxODUiIHkxPSIxNTUiIHgyPSIxOTUiIHkyPSIxODUiIHN0cm9rZT0iIzZiNWE0NSIgc3Ryb2tlLXdpZHRoPSIzIi8+CiAgICA8Y2lyY2xlIGN4PSIxNzUiIGN5PSIxOTAiIHI9IjUiIGZpbGw9IiM2YjVhNDUiLz4KICAgIDxjaXJjbGUgY3g9IjE5NSIgY3k9IjE5MCIgcj0iNSIgZmlsbD0iIzZiNWE0NSIvPgogIDwvZz4KPC9zdmc+",
"reward": 1.0,
"structure": 1.0,
"judged": false,
"policy": "beam-low"
},
{
"svg_b64": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MjAgMzIwIiB3aWR0aD0iNDIwIiBoZWlnaHQ9IjMyMCI+CiAgPGcgc3Ryb2tlPSIjMjIyIiBzdHJva2Utd2lkdGg9IjQiIGZpbGw9Im5vbmUiPgogICAgPGNpcmNsZSBjeD0iMTEwIiBjeT0iMjM1IiByPSI1OCIvPgogICAgPGNpcmNsZSBjeD0iMzEwIiBjeT0iMjM1IiByPSI1OCIvPgogICAgPGxpbmUgeDE9IjExMCIgeTE9IjIzNSIgeDI9IjIxMCIgeTI9IjE1MCIvPgogICAgPGxpbmUgeDE9IjMxMCIgeTE9IjIzNSIgeDI9IjIxMCIgeTI9IjE1MCIvPgogICAgPGxpbmUgeDE9IjIxMCIgeTE9IjE1MCIgeDI9IjIxMCIgeTI9IjExNSIvPgogICAgPGxpbmUgeDE9IjMxMCIgeTE9IjIzNSIgeDI9IjMxNSIgeTI9IjE4NSIvPgogICAgPGxpbmUgeDE9IjMxNSIgeTE9IjE4NSIgeDI9IjMwMCIgeTI9IjE1MCIvPgogICAgPGxpbmUgeDE9IjMwMCIgeTE9IjE1MCIgeDI9IjI4NSIgeTI9IjE0NSIvPgogIDwvZz4KICA8ZWxsaXBzZSBjeD0iMTk1IiBjeT0iMTI1IiByeD0iNjIiIHJ5PSI0MiIgZmlsbD0iI2YyZjJmMiIgc3Ryb2tlPSIjMjIyIiBzdHJva2Utd2lkdGg9IjMiLz4KICA8Y2lyY2xlIGN4PSIyMzUiIGN5PSI5OCIgcj0iMjgiIGZpbGw9IiNmMmYyZjIiIHN0cm9rZT0iIzIyMiIgc3Ryb2tlLXdpZHRoPSIzIi8+CiAgPHBhdGggZD0iTTI1NSA5MiBMMzQ1IDEwMiBMMjU1IDEyNCBaIiBmaWxsPSIjZjdjOTQ4IiBzdHJva2U9IiMyMjIiIHN0cm9rZS13aWR0aD0iMyIvPgogIDxlbGxpcHNlIGN4PSIyNTUiIGN5PSIxNDIiIHJ4PSIzNCIgcnk9IjI1IiBmaWxsPSIjZThiODlhIiBzdHJva2U9IiMyMjIiIHN0cm9rZS13aWR0aD0iMyIvPgogIDxjaXJjbGUgY3g9IjI0NiIgY3k9IjkyIiByPSI0IiBmaWxsPSIjMjIyIi8+CiAgPHBhdGggZD0iTTE0NSAxMjAgUTE3NSAxNjUgMjE1IDEyNSIgZmlsbD0iI2Q5ZDlkOSIgc3Ryb2tlPSIjMjIyIiBzdHJva2Utd2lkdGg9IjMiLz4KICA8cmVjdCB4PSIxOTUiIHk9IjE0OCIgd2lkdGg9IjgiIGhlaWdodD0iMzYiIGZpbGw9IiM2NjYiLz4KICA8ZWxsaXBzZSBjeD0iMzEwIiBjeT0iMjM1IiByeD0iOCIgcnk9IjgiIGZpbGw9IiMyMjIiLz4KICA8ZWxsaXBzZSBjeD0iMTEwIiBjeT0iMjM1IiByeD0iOCIgcnk9IjgiIGZpbGw9IiMyMjIiLz4KPC9zdmc+",
"reward": 1.0,
"structure": 1.0,
"judged": false,
"policy": "beam-low"
},
{
"svg_b64": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI0MDAiIGhlaWdodD0iMjIwIiB2aWV3Qm94PSIwIDAgNDAwIDIyMCI+CiAgPGNpcmNsZSBjeD0iMTAwIiBjeT0iMTcwIiByPSI0NSIgZmlsbD0ibm9uZSIgc3Ryb2tlPSJibGFjayIgc3Ryb2tlLXdpZHRoPSI1Ii8+CiAgPGNpcmNsZSBjeD0iMTAwIiBjeT0iMTcwIiByPSI2IiBmaWxsPSJibGFjayIvPgogIDxjaXJjbGUgY3g9IjI4MCIgY3k9IjE3MCIgcj0iNDUiIGZpbGw9Im5vbmUiIHN0cm9rZT0iYmxhY2siIHN0cm9rZS13aWR0aD0iNSIvPgogIDxjaXJjbGUgY3g9IjI4MCIgY3k9IjE3MCIgcj0iNiIgZmlsbD0iYmxhY2siLz4KICA8bGluZSB4MT0iMTAwIiB5MT0iMTcwIiB4Mj0iMTcwIiB5Mj0iMTIwIiBzdHJva2U9ImJsYWNrIiBzdHJva2Utd2lkdGg9IjUiLz4KICA8bGluZSB4MT0iMTcwIiB5MT0iMTIwIiB4Mj0iMjgwIiB5Mj0iMTcwIiBzdHJva2U9ImJsYWNrIiBzdHJva2Utd2lkdGg9IjUiLz4KICA8bGluZSB4MT0iMTcwIiB5MT0iMTIwIiB4Mj0iMTkwIiB5Mj0iMTAwIiBzdHJva2U9ImJsYWNrIiBzdHJva2Utd2lkdGg9IjUiLz4KICA8bGluZSB4MT0iMTkwIiB5MT0iMTAwIiB4Mj0iMjMwIiB5Mj0iODUiIHN0cm9rZT0iYmxhY2siIHN0cm9rZS13aWR0aD0iNSIvPgogIDxsaW5lIHgxPSIxOTAiIHkxPSIxMDAiIHgyPSIyMTAiIHkyPSI4MCIgc3Ryb2tlPSJibGFjayIgc3Ryb2tlLXdpZHRoPSI0Ii8+CiAgPHJlY3QgeD0iMTU1IiB5PSIxMTAiIHdpZHRoPSIzMCIgaGVpZ2h0PSI4IiBmaWxsPSJzYWRkbGVicm93biIvPgogIDxlbGxpcHNlIGN4PSIxNzAiIGN5PSI3NSIgcng9IjQ1IiByeT0iMjgiIGZpbGw9IndoaXRlIiBzdHJva2U9ImJsYWNrIiBzdHJva2Utd2lkdGg9IjIiLz4KICA8Y2lyY2xlIGN4PSIxMzUiIGN5PSI1MCIgcj0iMjQiIGZpbGw9IndoaXRlIiBzdHJva2U9ImJsYWNrIiBzdHJva2Utd2lkdGg9IjIiLz4KICA8cG9seWdvbiBwb2ludHM9IjExMSw0OCA3MCw1OCAxMTEsNjgiIGZpbGw9Im9yYW5nZSIgc3Ryb2tlPSJibGFjayIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgPGVsbGlwc2UgY3g9IjEwNSIgY3k9IjcyIiByeD0iMjAiIHJ5PSIxMiIgZmlsbD0ib3JhbmdlIiBzdHJva2U9ImJsYWNrIiBzdHJva2Utd2lkdGg9IjIiLz4KICA8ZWxsaXBzZSBjeD0iMTQwIiBjeT0iNzUiIHJ4PSI0IiByeT0iOCIgZmlsbD0iYmxhY2siLz4KICA8ZWxsaXBzZSBjeD0iMTU1IiBjeT0iOTUiIHJ4PSIxOCIgcnk9IjEwIiBmaWxsPSJibGFjayIvPgogIDxsaW5lIHgxPSIxOTUiIHkxPSI5NSIgeDI9IjIyNSIgeTI9IjE1MCIgc3Ryb2tlPSJibGFjayIgc3Ryb2tlLXdpZHRoPSI0Ii8+CiAgPGxpbmUgeDE9IjIyNSIgeTE9IjE1MCIgeDI9IjIzNSIgeTI9IjE2MCIgc3Ryb2tlPSJibGFjayIgc3Ryb2tlLXdpZHRoPSI0Ii8+CiAgPGxpbmUgeDE9IjE3NSIgeTE9Ijk4IiB4Mj0iMjAwIiB5Mj0iMTU1IiBzdHJva2U9ImJsYWNrIiBzdHJva2Utd2lkdGg9IjQiLz4KICA8bGluZSB4MT0iMjAwIiB5MT0iMTU1IiB4Mj0iMjE1IiB5Mj0iMTY1IiBzdHJva2U9ImJsYWNrIiBzdHJva2Utd2lkdGg9IjQiLz4KPC9zdmc+",
"reward": 1.0,
"structure": 1.0,
"judged": false,
"policy": "beam-low"
},
{
"svg_b64": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI1MTIiIGhlaWdodD0iMzg0IiB2aWV3Qm94PSIwIDAgNTEyIDM4NCI+CiAgPCEtLSBiaWN5Y2xlIC0tPgogIDxjaXJjbGUgY3g9IjEyMCIgY3k9IjI4MCIgcj0iNDgiIGZpbGw9Im5vbmUiIHN0cm9rZT0iIzMzMyIgc3Ryb2tlLXdpZHRoPSI2Ii8+CiAgPGNpcmNsZSBjeD0iMzYwIiBjeT0iMjgwIiByPSI0OCIgZmlsbD0ibm9uZSIgc3Ryb2tlPSIjMzMzIiBzdHJva2Utd2lkdGg9IjYiLz4KICA8bGluZSB4MT0iMTIwIiB5MT0iMjgwIiB4Mj0iMjQwIiB5Mj0iMjEwIiBzdHJva2U9IiMzMzMiIHN0cm9rZS13aWR0aD0iNiIvPgogIDxsaW5lIHgxPSIyNDAiIHkxPSIyMTAiIHgyPSIzNjAiIHkyPSIyODAiIHN0cm9rZT0iIzMzMyIgc3Ryb2tlLXdpZHRoPSI2Ii8+CiAgPGxpbmUgeDE9IjI0MCIgeTE9IjIxMCIgeDI9IjI0MCIgeTI9IjE1MCIgc3Ryb2tlPSIjMzMzIiBzdHJva2Utd2lkdGg9IjYiLz4KICA8bGluZSB4MT0iMjI1IiB5MT0iMTUwIiB4Mj0iMjc1IiB5Mj0iMTUwIiBzdHJva2U9IiMzMzMiIHN0cm9rZS13aWR0aD0iNiIvPgogIDxsaW5lIHgxPSIyNDAiIHkxPSIxNzAiIHgyPSIzMDAiIHkyPSIxNjAiIHN0cm9rZT0iIzMzMyIgc3Ryb2tlLXdpZHRoPSI1Ii8+CiAgPGNpcmNsZSBjeD0iMzA1IiBjeT0iMTYwIiByPSI4IiBmaWxsPSIjMzMzIi8+CiAgPCEtLSBwZWxpY2FuIGJvZHkgLS0+CiAgPGVsbGlwc2UgY3g9IjI1MCIgY3k9IjE0NSIgcng9IjU4IiByeT0iMzQiIGZpbGw9IiNmMmYyZjIiIHN0cm9rZT0iIzU1NSIgc3Ryb2tlLXdpZHRoPSIzIi8+CiAgPCEtLSBwZWxpY2FuIHRhaWwgLS0+CiAgPHBvbHlnb24gcG9pbnRzPSIxOTUsMTUwIDE2NSwxMzUgMTcwLDE2NSIgZmlsbD0iI2UwZTBlMCIgc3Ryb2tlPSIjNTU1IiBzdHJva2Utd2lkdGg9IjIiLz4KICA8IS0tIHBlbGljYW4gaGVhZCAtLT4KICA8Y2lyY2xlIGN4PSIzMDUiIGN5PSIxMDUiIHI9IjI2IiBmaWxsPSIjZjJmMmYyIiBzdHJva2U9IiM1NTUiIHN0cm9rZS13aWR0aD0iMyIvPgogIDwhLS0gbG9uZyBiZWFrIC0tPgogIDxwYXRoIGQ9Ik0zMjUsOTggQzM3NSw5MiA0MjUsODggNDY1LDkyIEw0NTAsMTEyIEM0MTAsMTA4IDM2NSwxMTIgMzI1LDExOCBaIiBmaWxsPSIjZjRjMDVhIiBzdHJva2U9IiNhNjdjMmQiIHN0cm9rZS13aWR0aD0iMiIvPgogIDwhLS0gdGhyb2F0IHBvdWNoIC0tPgogIDxwYXRoIGQ9Ik0zMjUsMTE4IEMzNDUsMTQwIDM3NSwxNDggMzk1LDEzOCBDMzgyLDEyNCAzNTUsMTE4IDMyNSwxMTggWiIgZmlsbD0iI2Q5OWE2YSIgc3Ryb2tlPSIjYTY3YzJkIiBzdHJva2Utd2lkdGg9IjIiLz4KICA8IS0tIGV5ZSAtLT4KICA8Y2lyY2xlIGN4PSIzMTAiIGN5PSIxMDAiIHI9IjQiIGZpbGw9IiMyMjIiLz4KICA8IS0tIG5lY2sgLS0+CiAgPHBhdGggZD0iTTI4NSwxMjUgQzI5NSwxMzggMjkwLDE1MCAyNzUsMTUwIiBmaWxsPSJub25lIiBzdHJva2U9IiM1NTUiIHN0cm9rZS13aWR0aD0iNCIvPgo8L3N2Zz4=",
"reward": 1.0,
"structure": 1.0,
"judged": false,
"policy": "beam-low"
},
{
"svg_b64": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI1MTIiIGhlaWdodD0iMzg0IiB2aWV3Qm94PSIwIDAgNTEyIDM4NCI+CiAgPCEtLSBiaWN5Y2xlIC0tPgogIDxjaXJjbGUgY3g9IjE1MCIgY3k9IjI4MCIgcj0iNjIiIGZpbGw9Im5vbmUiIHN0cm9rZT0iIzMzMyIgc3Ryb2tlLXdpZHRoPSI2Ii8+CiAgPGNpcmNsZSBjeD0iMzQwIiBjeT0iMjgwIiByPSI2MiIgZmlsbD0ibm9uZSIgc3Ryb2tlPSIjMzMzIiBzdHJva2Utd2lkdGg9IjYiLz4KICA8Y2lyY2xlIGN4PSIxNTAiIGN5PSIyODAiIHI9IjE0IiBmaWxsPSIjMzMzIi8+CiAgPGNpcmNsZSBjeD0iMzQwIiBjeT0iMjgwIiByPSIxNCIgZmlsbD0iIzMzMyIvPgogIDxsaW5lIHgxPSIxNTAiIHkxPSIyODAiIHgyPSIyMjgiIHkyPSIyMjIiIHN0cm9rZT0iIzMzMyIgc3Ryb2tlLXdpZHRoPSI3Ii8+CiAgPGxpbmUgeDE9IjIyOCIgeTE9IjIyMiIgeDI9IjI4MiIgeTI9IjIyMiIgc3Ryb2tlPSIjMzMzIiBzdHJva2Utd2lkdGg9IjciLz4KICA8bGluZSB4MT0iMjgyIiB5MT0iMjIyIiB4Mj0iMzQwIiB5Mj0iMjgwIiBzdHJva2U9IiMzMzMiIHN0cm9rZS13aWR0aD0iNyIvPgogIDxsaW5lIHgxPSIyMjgiIHkxPSIyMjIiIHgyPSIyMjgiIHkyPSIyNjgiIHN0cm9rZT0iIzMzMyIgc3Ryb2tlLXdpZHRoPSI3Ii8+CiAgPGxpbmUgeDE9IjIyOCIgeTE9IjI2OCIgeDI9IjM0MCIgeTI9IjI4MCIgc3Ryb2tlPSIjMzMzIiBzdHJva2Utd2lkdGg9IjciLz4KICA8bGluZSB4MT0iMjgyIiB5MT0iMjIyIiB4Mj0iMjgyIiB5Mj0iMTg0IiBzdHJva2U9IiMzMzMiIHN0cm9rZS13aWR0aD0iNiIvPgogIDxsaW5lIHgxPSIyODIiIHkxPSIxODQiIHgyPSIzMTgiIHkyPSIxNzYiIHN0cm9rZT0iIzMzMyIgc3Ryb2tlLXdpZHRoPSI2Ii8+CiAgPGNpcmNsZSBjeD0iMjI4IiBjeT0iMjY4IiByPSIxMiIgZmlsbD0iIzMzMyIvPgogIDxsaW5lIHgxPSIyMjgiIHkxPSIyMjIiIHgyPSIyNDgiIHkyPSIxODgiIHN0cm9rZT0iIzMzMyIgc3Ryb2tlLXdpZHRoPSI1Ii8+CiAgPHJlY3QgeD0iMjM2IiB5PSIxODAiIHdpZHRoPSIzNCIgaGVpZ2h0PSIxMCIgcng9IjUiIGZpbGw9IiM1NTUiLz4KCiAgPCEtLSBwZWxpY2FuIC0tPgogIDxlbGxpcHNlIGN4PSIyNTAiIGN5PSIxODYiIHJ4PSI0OCIgcnk9IjM0IiBmaWxsPSIjZjVmMGQ4IiBzdHJva2U9IiM5OTkiIHN0cm9rZS13aWR0aD0iMyIvPgogIDxlbGxpcHNlIGN4PSIzMDIiIGN5PSIxNDYiIHJ4PSIzMiIgcnk9IjI2IiBmaWxsPSIjZjVmMGQ4IiBzdHJva2U9IiM5OTkiIHN0cm9rZS13aWR0aD0iMyIvPgogIDxwb2x5Z29uIHBvaW50cz0iMjcwLDE0MiAzNjAsMTI4IDQwMiwxNDUgMzU4LDE1OCAyNzAsMTU2IiBmaWxsPSIjZmZjYzY2IiBzdHJva2U9IiNiODg2MGIiIHN0cm9rZS13aWR0aD0iMyIvPgogIDxwYXRoIGQ9Ik0yODggMTU4IFEzMDggMTk0IDM0OCAxOTAgUTM3NiAxODYgMzYwIDE1NCIgZmlsbD0iI2U4YjA0YSIgc3Ryb2tlPSIjYjg4NjBiIiBzdHJva2Utd2lkdGg9IjMiLz4KICA8Y2lyY2xlIGN4PSIzMjAiIGN5PSIxMzQiIHI9IjYiIGZpbGw9IiMxMTEiLz4KICA8cGF0aCBkPSJNMjA2IDE4NiBRMTgyIDIwNiAxNzAgMjM4IFEyMDYgMjM0IDIzMCAyMTAgWiIgZmlsbD0iI2Q4Y2ZiMCIgc3Ryb2tlPSIjOTk5IiBzdHJva2Utd2lkdGg9IjIiLz4KICA8cGF0aCBkPSJNMjM1IDIxNCBRMjU1IDIzOCAyODIgMjMyIFEzMDIgMjI4IDI5NiAyMDgiIGZpbGw9IiNkOGNmYjAiIHN0cm9rZT0iIzk5OSIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgPGxpbmUgeDE9IjI0OCIgeTE9IjIxNCIgeDI9IjI0NCIgeTI9IjI1OCIgc3Ryb2tlPSIjOTk5IiBzdHJva2Utd2lkdGg9IjUiLz4KICA8bGluZSB4MT0iMjY2IiB5MT0iMjE2IiB4Mj0iMjY0IiB5Mj0iMjY4IiBzdHJva2U9IiM5OTkiIHN0cm9rZS13aWR0aD0iNSIvPgogIDxsaW5lIHgxPSIyNDQiIHkxPSIyNTgiIHgyPSIyMzAiIHkyPSIyODAiIHN0cm9rZT0iIzk5OSIgc3Ryb2tlLXdpZHRoPSI1Ii8+CiAgPGxpbmUgeDE9IjI2NCIgeTE9IjI2OCIgeDI9IjI3OCIgeTI9IjI4NiIgc3Ryb2tlPSIjOTk5IiBzdHJva2Utd2lkdGg9IjUiLz4KICA8bGluZSB4MT0iMjc4IiB5MT0iMjg2IiB4Mj0iMzEyIiB5Mj0iMjgwIiBzdHJva2U9IiM5OTkiIHN0cm9rZS13aWR0aD0iNSIvPgo8L3N2Zz4=",
"reward": 1.0,
"structure": 1.0,
"judged": false,
"policy": "beam-low"
},
{
"svg_b64": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MDAgMzAwIiB3aWR0aD0iNDAwIiBoZWlnaHQ9IjMwMCI+CiAgPCEtLSBiaWN5Y2xlIC0tPgogIDxnIGlkPSJiaWtlIj4KICAgIDwhLS0gd2hlZWxzIC0tPgogICAgPGNpcmNsZSBjeD0iMTIwIiBjeT0iMjQwIiByPSI0NSIgZmlsbD0ibm9uZSIgc3Ryb2tlPSIjMzMzIiBzdHJva2Utd2lkdGg9IjYiLz4KICAgIDxjaXJjbGUgY3g9IjEyMCIgY3k9IjI0MCIgcj0iMzAiIGZpbGw9Im5vbmUiIHN0cm9rZT0iIzg4OCIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgICA8Y2lyY2xlIGN4PSIyODAiIGN5PSIyNDAiIHI9IjQ1IiBmaWxsPSJub25lIiBzdHJva2U9IiMzMzMiIHN0cm9rZS13aWR0aD0iNiIvPgogICAgPGNpcmNsZSBjeD0iMjgwIiBjeT0iMjQwIiByPSIzMCIgZmlsbD0ibm9uZSIgc3Ryb2tlPSIjODg4IiBzdHJva2Utd2lkdGg9IjIiLz4KICAgIDwhLS0gZnJhbWUgLS0+CiAgICA8bGluZSB4MT0iMTIwIiB5MT0iMjQwIiB4Mj0iMjAwIiB5Mj0iMTgwIiBzdHJva2U9IiNjMDM5MmIiIHN0cm9rZS13aWR0aD0iNSIvPgogICAgPGxpbmUgeDE9IjIwMCIgeTE9IjE4MCIgeDI9IjI4MCIgeTI9IjI0MCIgc3Ryb2tlPSIjYzAzOTJiIiBzdHJva2Utd2lkdGg9IjUiLz4KICAgIDxsaW5lIHgxPSIyMDAiIHkxPSIxODAiIHgyPSIyMDAiIHkyPSIxNTAiIHN0cm9rZT0iI2MwMzkyYiIgc3Ryb2tlLXdpZHRoPSI1Ii8+CiAgICA8bGluZSB4MT0iMTYwIiB5MT0iMjQwIiB4Mj0iMjAwIiB5Mj0iMTgwIiBzdHJva2U9IiNjMDM5MmIiIHN0cm9rZS13aWR0aD0iNSIvPgogICAgPCEtLSBoYW5kbGViYXJzIC0tPgogICAgPGxpbmUgeDE9IjIwMCIgeTE9IjE1MCIgeDI9IjIxNSIgeTI9IjEzNSIgc3Ryb2tlPSIjMzMzIiBzdHJva2Utd2lkdGg9IjQiLz4KICAgIDxsaW5lIHgxPSIyMTUiIHkxPSIxMzUiIHgyPSIyMzAiIHkyPSIxMzUiIHN0cm9rZT0iIzMzMyIgc3Ryb2tlLXdpZHRoPSI0Ii8+CiAgICA8IS0tIHNhZGRsZSAtLT4KICAgIDxyZWN0IHg9IjE5MCIgeT0iMTQ1IiB3aWR0aD0iMzAiIGhlaWdodD0iOCIgcng9IjMiIGZpbGw9IiM1NTUiLz4KICAgIDwhLS0gcGVkYWxzIC0tPgogICAgPGNpcmNsZSBjeD0iMjAwIiBjeT0iMjEwIiByPSI2IiBmaWxsPSIjMzMzIi8+CiAgPC9nPgogIDwhLS0gcGVsaWNhbiAtLT4KICA8ZyBpZD0icGVsaWNhbiI+CiAgICA8IS0tIGJvZHkgLS0+CiAgICA8ZWxsaXBzZSBjeD0iMjAwIiBjeT0iMTMwIiByeD0iMzUiIHJ5PSIyMiIgZmlsbD0iI2Y0ZjRmNCIgc3Ryb2tlPSIjOTk5IiBzdHJva2Utd2lkdGg9IjEuNSIvPgogICAgPCEtLSB3aW5nIC0tPgogICAgPGVsbGlwc2UgY3g9IjE4NSIgY3k9IjEzMiIgcng9IjE4IiByeT0iOCIgZmlsbD0iI2QwZDBkMCIgc3Ryb2tlPSIjOTk5IiBzdHJva2Utd2lkdGg9IjEiLz4KICAgIDwhLS0gbmVjayAtLT4KICAgIDxwYXRoIGQ9Ik0gMjI1IDEyMCBRIDI2MCA5MCAyNjUgNjAiIHN0cm9rZT0iI2Y0ZjRmNCIgc3Ryb2tlLXdpZHRoPSIxMiIgZmlsbD0ibm9uZSIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIi8+CiAgICA8IS0tIGhlYWQgLS0+CiAgICA8Y2lyY2xlIGN4PSIyNjgiIGN5PSI1OCIgcj0iMTQiIGZpbGw9IiNmNGY0ZjQiIHN0cm9rZT0iIzk5OSIgc3Ryb2tlLXdpZHRoPSIxLjUiLz4KICAgIDwhLS0gYmVhayBsb25nIC0tPgogICAgPHBhdGggZD0iTSAyNzggNTUgTCAzMzUgNDggTCAzMzUgNjIgTCAyNzggNjUgWiIgZmlsbD0iI2YwYjQyOSIgc3Ryb2tlPSIjYzQ4YTAwIiBzdHJva2Utd2lkdGg9IjEiLz4KICAgIDwhLS0gdGhyb2F0IHBvdWNoIHVuZGVyIGJlYWsgLS0+CiAgICA8cGF0aCBkPSJNIDI4NSA2NSBRIDMxNSA3OCAzMzAgNjIiIGZpbGw9IiNmMGI0MjkiIHN0cm9rZT0iI2M0OGEwMCIgc3Ryb2tlLXdpZHRoPSIxIi8+CiAgICA8IS0tIGV5ZSAtLT4KICAgIDxjaXJjbGUgY3g9IjI3NSIgY3k9IjUyIiByPSIyLjUiIGZpbGw9IiMyMjIiLz4KICAgIDwhLS0gbGVncyAtLT4KICAgIDxsaW5lIHgxPSIxODUiIHkxPSIxNDgiIHgyPSIxODAiIHkyPSIyMDUiIHN0cm9rZT0iI2YwYjQyOSIgc3Ryb2tlLXdpZHRoPSIzIi8+CiAgICA8bGluZSB4MT0iMjEwIiB5MT0iMTQ4IiB4Mj0iMjA1IiB5Mj0iMjA1IiBzdHJva2U9IiNmMGI0MjkiIHN0cm9rZS13aWR0aD0iMyIvPgogICAgPCEtLSBmZWV0IC0tPgogICAgPGVsbGlwc2UgY3g9IjE3NSIgY3k9IjIwOCIgcng9IjgiIHJ5PSIzIiBmaWxsPSIjZjBiNDI5Ii8+CiAgICA8ZWxsaXBzZSBjeD0iMjAwIiBjeT0iMjA4IiByeD0iOCIgcnk9IjMiIGZpbGw9IiNmMGI0MjkiLz4KICA8L2c+Cjwvc3ZnPg==",
"reward": 1.0,
"structure": 1.0,
"judged": false,
"policy": "beam-low"
},
{
"svg_b64": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI0MjAiIGhlaWdodD0iMzIwIiB2aWV3Qm94PSIwIDAgNDIwIDMyMCI+CiAgPHJlY3Qgd2lkdGg9IjQyMCIgaGVpZ2h0PSIzMjAiIGZpbGw9IiNiZmU3ZmYiLz4KICA8ZWxsaXBzZSBjeD0iMjEwIiBjeT0iMzAwIiByeD0iMTkwIiByeT0iMjIiIGZpbGw9IiM3ZmM5N2YiLz4KICA8IS0tIGJpY3ljbGUgd2hlZWxzIC0tPgogIDxjaXJjbGUgY3g9IjExNSIgY3k9IjIzNSIgcj0iNTIiIGZpbGw9Im5vbmUiIHN0cm9rZT0iIzIyMiIgc3Ryb2tlLXdpZHRoPSI4Ii8+CiAgPGNpcmNsZSBjeD0iMTE1IiBjeT0iMjM1IiByPSI4IiBmaWxsPSIjNTU1Ii8+CiAgPGNpcmNsZSBjeD0iMzA1IiBjeT0iMjM1IiByPSI1MiIgZmlsbD0ibm9uZSIgc3Ryb2tlPSIjMjIyIiBzdHJva2Utd2lkdGg9IjgiLz4KICA8Y2lyY2xlIGN4PSIzMDUiIGN5PSIyMzUiIHI9IjgiIGZpbGw9IiM1NTUiLz4KICA8IS0tIGJpY3ljbGUgZnJhbWUgLS0+CiAgPHBvbHlnb24gcG9pbnRzPSIxMTUsMjM1IDIwNSwxNjUgMzA1LDIzNSAyMTUsMjM1IiBmaWxsPSJub25lIiBzdHJva2U9IiMyYjVmOGYiIHN0cm9rZS13aWR0aD0iNyIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIvPgogIDxsaW5lIHgxPSIyMDUiIHkxPSIxNjUiIHgyPSIyMzUiIHkyPSIyMDUiIHN0cm9rZT0iIzJiNWY4ZiIgc3Ryb2tlLXdpZHRoPSI3Ii8+CiAgPGxpbmUgeDE9IjIzNSIgeTE9IjIwNSIgeDI9IjMwNSIgeTI9IjIzNSIgc3Ryb2tlPSIjMmI1ZjhmIiBzdHJva2Utd2lkdGg9IjciLz4KICA8IS0tIHNlYXQgLS0+CiAgPGVsbGlwc2UgY3g9IjIxNSIgY3k9IjE1OCIgcng9IjI0IiByeT0iMTAiIGZpbGw9IiM2YjRmMmEiLz4KICA8IS0tIGhhbmRsZWJhcnMgLS0+CiAgPGxpbmUgeDE9IjMwNSIgeTE9IjIzNSIgeDI9IjMwNSIgeTI9IjE2MCIgc3Ryb2tlPSIjMmI1ZjhmIiBzdHJva2Utd2lkdGg9IjciLz4KICA8cG9seWxpbmUgcG9pbnRzPSIyODgsMTYwIDMyMiwxNjAgMzIyLDE0NSIgZmlsbD0ibm9uZSIgc3Ryb2tlPSIjMjIyIiBzdHJva2Utd2lkdGg9IjYiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIvPgogIDwhLS0gcGVsaWNhbiBib2R5IC0tPgogIDxlbGxpcHNlIGN4PSIyMjUiIGN5PSIxMjgiIHJ4PSI1OCIgcnk9IjM2IiBmaWxsPSIjZjdmMmU3IiBzdHJva2U9IiM5OTkiIHN0cm9rZS13aWR0aD0iMiIvPgogIDxlbGxpcHNlIGN4PSIxOTUiIGN5PSIxMjIiIHJ4PSIyNCIgcnk9IjE4IiBmaWxsPSIjZmZmZmZmIiBvcGFjaXR5PSIwLjU1Ii8+CiAgPCEtLSB3aW5nIC0tPgogIDxwYXRoIGQ9Ik0xOTUsMTMyIEMyMDUsMTUwIDI1MCwxNTUgMjc1LDEzNSBDMjU1LDEzOCAyMjUsMTQyIDE5NSwxMzIgWiIgZmlsbD0iI2RkZDZjNCIvPgogIDwhLS0gbmVjayAtLT4KICA8cGF0aCBkPSJNMjY1LDExOCBDMjkwLDg1IDMwNSw3MCAzMTUsNjIiIGZpbGw9Im5vbmUiIHN0cm9rZT0iI2Y3ZjJlNyIgc3Ryb2tlLXdpZHRoPSIxOCIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIi8+CiAgPCEtLSBoZWFkIC0tPgogIDxlbGxpcHNlIGN4PSIzMjUiIGN5PSI1MiIgcng9IjI2IiByeT0iMTgiIGZpbGw9IiNmN2YyZTciIHN0cm9rZT0iIzk5OSIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgPGNpcmNsZSBjeD0iMzMzIiBjeT0iNDciIHI9IjMuNSIgZmlsbD0iIzExMSIvPgogIDwhLS0gbG9uZyBiZWFrIC0tPgogIDxwb2x5Z29uIHBvaW50cz0iMzQ2LDQ4IDQwNSw1OCAzNDYsNjgiIGZpbGw9IiNmMGEzM2IiIHN0cm9rZT0iI2I2NzMxOCIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgPCEtLSB0aHJvYXQgcG91Y2ggdW5kZXIgYmVhayAtLT4KICA8ZWxsaXBzZSBjeD0iMzUwIiBjeT0iNzYiIHJ4PSIzNCIgcnk9IjIwIiBmaWxsPSIjZjdkNmEzIiBzdHJva2U9IiNiNjczMTgiIHN0cm9rZS13aWR0aD0iMiIvPgogIDxwYXRoIGQ9Ik0zMzQsNjYgQzM0OCw4NCAzNzAsODYgMzg2LDcwIiBmaWxsPSJub25lIiBzdHJva2U9IiNkOThiMmIiIHN0cm9rZS13aWR0aD0iMiIvPgogIDwhLS0gbGVncyBhbmQgcGVkYWxzIC0tPgogIDxsaW5lIHgxPSIyMDUiIHkxPSIxNTYiIHgyPSIxODUiIHkyPSIxODUiIHN0cm9rZT0iI2NmYTA1YSIgc3Ryb2tlLXdpZHRoPSI1IiBzdHJva2UtbGluZWNhcD0icm91bmQiLz4KICA8bGluZSB4MT0iMjQ1IiB5MT0iMTU2IiB4Mj0iMjY1IiB5Mj0iMTg1IiBzdHJva2U9IiNjZmEwNWEiIHN0cm9rZS13aWR0aD0iNSIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIi8+CiAgPGNpcmNsZSBjeD0iMjI4IiBjeT0iMjA1IiByPSIxMiIgZmlsbD0iIzQ0NCIvPgogIDxlbGxpcHNlIGN4PSIyMjgiIGN5PSIyMDUiIHJ4PSI1IiByeT0iNSIgZmlsbD0iIzk5OSIvPgogIDwhLS0gc21hbGwgdGFpbCAtLT4KICA8cGF0aCBkPSJNMTcwLDEyOCBDMTQ1LDEyMCAxNDAsMTQ1IDE2MiwxNTIiIGZpbGw9IiNkZGQ2YzQiLz4KPC9zdmc+",
"reward": 1.0,
"structure": 1.0,
"judged": false,
"policy": "beam-low"
}
],
"links": {
"dataset": "https://huggingface.co/datasets/JagritGumber/shortlist",
"code": "https://huggingface.co/datasets/JagritGumber/shortlist/tree/main/code",
"patch": "https://huggingface.co/datasets/JagritGumber/shortlist/blob/main/fixes/connect4_env.patch",
"training": "https://huggingface.co/datasets/JagritGumber/shortlist/blob/main/training/RESULTS.md",
"rfc": "https://github.com/openenv/OpenEnv/blob/main/rfcs/008-environment-auto-validation.md"
}
}
{
"numbers": {
"environments": 32,
"task_suites": 47,
"community_environments": 20,
"episodes": 1177,
"beam_episodes": 640,
"small_episodes": 240,
"beam_output_tokens": 1572150,
"suites_played_by": {
"null": 47,
"random": 14,
"small": 10,
"beam-low": 46,
"beam-medium": 6,
"beam-high": 12,
"beam-max": 1
}
},
"verdicts": [
{
"suite": "chess_env",
"env": "chess_env",
"notes": "vs moonfish depth 2; +1 win, -1 loss; capped at 30 moves",
"episodes": 14,
"errors": 0,
"harness_errors": 0,
"tiers": {
"beam-low": {
"episodes": 6,
"mean": -0.8333333333333334,
"std": 0.37267799624996495,
"within_task_std": 0.23570226039551584,
"grpo_signal": 0.5,
"out_tokens": 49037,
"invalid_steps": 0,
"steps": 108,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 3
},
"null": {
"episodes": 2,
"mean": -1.0,
"std": 0.0,
"within_task_std": null,
"grpo_signal": null,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 18,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 1
},
"random": {
"episodes": 6,
"mean": -1.0,
"std": 0.0,
"within_task_std": 0.0,
"grpo_signal": 0.0,
"out_tokens": 0,
"invalid_steps": 0,
"steps": 100,
"pass@1": 0.0,
"pass@k": 0.0,
"k": 3
}
},
"checks": {
"semantic.floor_gap": {
"pass": true,
"floor_solve_rate": 0.0,
"floor_policy": "null"
},
"statistical.reward_reachability": {
"pass": false,
"reference": "beam-low",
"pass@1": 0.0,
"pass@k": 0.0,
"k": 3
},
"statistical.difficulty_separation": {
"pass": false,
"reference_minus_floor": 0.16666666666666663,
"ci95": [
0.0,
0.5
],
"p": 0.33325,
"n": [
6,
2
]
},
"statistical.headroom": {
"pass": true,
"best_mean": -0.8333333333333334,
"max": 1.0,
"room_fraction": 0.9166666666666667
},