{ "numbers": { "environments": 32, "task_suites": 47, "community_environments": 20, "episodes": 1177, "beam_episodes": 640, "small_episodes": 240, "beam_output_tokens": 1572150, "suites_played_by": { "null": 47, "random": 14, "small": 10, "beam-low": 46, "beam-medium": 6, "beam-high": 12, "beam-max": 1 } }, "verdicts": [ { "suite": "chess_env", "env": "chess_env", "notes": "vs moonfish depth 2; +1 win, -1 loss; capped at 30 moves", "episodes": 14, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": -0.8333333333333334, "std": 0.37267799624996495, "within_task_std": 0.23570226039551584, "grpo_signal": 0.5, "out_tokens": 49037, "invalid_steps": 0, "steps": 108, "pass@1": 0.0, "pass@k": 0.0, "k": 3 }, "null": { "episodes": 2, "mean": -1.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 18, "pass@1": 0.0, "pass@k": 0.0, "k": 1 }, "random": { "episodes": 6, "mean": -1.0, "std": 0.0, "within_task_std": 0.0, "grpo_signal": 0.0, "out_tokens": 0, "invalid_steps": 0, "steps": 100, "pass@1": 0.0, "pass@k": 0.0, "k": 3 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_solve_rate": 0.0, "floor_policy": "null" }, "statistical.reward_reachability": { "pass": false, "reference": "beam-low", "pass@1": 0.0, "pass@k": 0.0, "k": 3 }, "statistical.difficulty_separation": { "pass": false, "reference_minus_floor": 0.16666666666666663, "ci95": [ 0.0, 0.5 ], "p": 0.33325, "n": [ 6, 2 ] }, "statistical.headroom": { "pass": true, "best_mean": -0.8333333333333334, "max": 1.0, "room_fraction": 0.9166666666666667 }, "statistical.variance_bound": { "pass": false, "within_task_std": 0.23570226039551584, "signal": 0.16666666666666663 }, "statistical.grpo_signal": { "pass": true, "reference": "beam-low", "share": 0.5, "threshold": 0.2 } }, "verdict": "out of reach: no player tested ever wins, Beam included", "group": "reach", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 2, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": -1.0, "solved": 0.0, "grpo": null, "n": 2 }, { "policy": "random", "x": 0.0, "mean": -1.0, "solved": 0.0, "grpo": 0.0, "n": 6 }, { "policy": "beam-low", "x": 0.0833, "mean": -0.8333333333333334, "solved": 0.0, "grpo": 0.5, "n": 6 } ], "top_known": true } }, { "suite": "connect4_env", "env": "connect4_env", "notes": "as shipped: no opponent is ever used", "episodes": 21, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 9, "mean": 1.0, "std": 0.0, "within_task_std": 0.0, "grpo_signal": 0.0, "out_tokens": 124935, "invalid_steps": 0, "steps": 151, "pass@1": 1.0, "pass@k": 1.0, "k": 3 }, "null": { "episodes": 3, "mean": 1.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 57, "pass@1": 1.0, "pass@k": 1.0, "k": 1 }, "random": { "episodes": 9, "mean": 1.0, "std": 0.0, "within_task_std": 0.0, "grpo_signal": 0.0, "out_tokens": 0, "invalid_steps": 0, "steps": 206, "pass@1": 1.0, "pass@k": 1.0, "k": 3 } }, "checks": { "semantic.floor_gap": { "pass": false, "floor_solve_rate": 1.0 } }, "verdict": "broken as shipped: doing nothing already earns the top reward", "defect": "No opponent: the agent moves for both sides, so every finished game pays +1.", "group": "broken", "rail": { "rungs": [ { "policy": "null", "x": 1.0, "mean": 1.0, "solved": 1.0, "grpo": null, "n": 3 }, { "policy": "random", "x": 1.0, "mean": 1.0, "solved": 1.0, "grpo": 0.0, "n": 9 }, { "policy": "beam-low", "x": 1.0, "mean": 1.0, "solved": 1.0, "grpo": 0.0, "n": 9 } ], "top_known": true } }, { "suite": "connect4_env-patched", "env": "connect4_env", "notes": "with Beam's fix: a built-in opponent, +1 only for an agent win", "episodes": 21, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 9, "mean": 0.7777777777777778, "std": 0.6285393610547089, "within_task_std": 0.31426968052735443, "grpo_signal": 0.3333333333333333, "out_tokens": 43094, "invalid_steps": 0, "steps": 62, "pass@1": 0.8888888888888888, "pass@k": 1.0, "k": 3 }, "null": { "episodes": 3, "mean": -1.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 15, "pass@1": 0.0, "pass@k": 0.0, "k": 1 }, "random": { "episodes": 9, "mean": -0.7777777777777778, "std": 0.6285393610547089, "within_task_std": 0.31426968052735443, "grpo_signal": 0.3333333333333333, "out_tokens": 0, "invalid_steps": 0, "steps": 56, "pass@1": 0.1111111111111111, "pass@k": 0.3333333333333333, "k": 3 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_solve_rate": 0.1111111111111111, "floor_policy": "random" }, "statistical.reward_reachability": { "pass": true, "reference": "beam-low", "pass@1": 0.8888888888888888, "pass@k": 1.0, "k": 3 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 1.5555555555555556, "ci95": [ 0.8888888888888888, 2.0 ], "p": 0.0, "n": [ 9, 9 ] }, "statistical.headroom": { "pass": true, "best_mean": 0.7777777777777778, "max": 1.0, "room_fraction": 0.1111111111111111 }, "statistical.variance_bound": { "pass": true, "within_task_std": 0.31426968052735443, "signal": 1.5555555555555556 }, "statistical.grpo_signal": { "pass": true, "reference": "beam-low", "share": 0.3333333333333333, "threshold": 0.2 } }, "verdict": "learnable for Beam; not yet tested with a small model", "group": "worth", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": -1.0, "solved": 0.0, "grpo": null, "n": 3 }, { "policy": "random", "x": 0.1111, "mean": -0.7777777777777778, "solved": 0.1111111111111111, "grpo": 0.3333333333333333, "n": 9 }, { "policy": "beam-low", "x": 0.8889, "mean": 0.7777777777777778, "solved": 0.8888888888888888, "grpo": 0.3333333333333333, "n": 9 } ], "top_known": true } }, { "suite": "dipg_safety_env", "env": "dipg_safety_env", "notes": "single medical-safety answer", "episodes": 16, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 12, "mean": -10.0, "std": 0.0, "within_task_std": 0.0, "grpo_signal": 0.0, "out_tokens": 2644, "invalid_steps": 0, "steps": 12 }, "null": { "episodes": 4, "mean": -10.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 4 } }, "checks": { "semantic.floor_gap": { "pass": false, "floor_mean": -10.0, "top": -10.0, "gap_fraction": 0.0 }, "statistical.reward_reachability": { "pass": false, "reference": "beam-low", "gain_over_floor": 0.0, "ci95": [ 0.0, 0.0 ], "p": 1.0 }, "statistical.difficulty_separation": { "pass": false, "reference_minus_floor": 0.0, "ci95": [ 0.0, 0.0 ], "p": 1.0, "n": [ 12, 4 ] }, "statistical.variance_bound": { "pass": true, "within_task_std": 0.0, "signal": 1e-09 }, "statistical.grpo_signal": { "pass": false, "reference": "beam-low", "share": 0.0, "threshold": 0.2 } }, "verdict": "broken as shipped: its default answer markers are chat-control tokens for Harmony-format models such as Beam, so no answer clears the format gate", "defect": "Its default answer markers are chat-control tokens for Harmony-format models such as Beam, so no answer clears the format gate.", "group": "broken", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 4, "provisional": false, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": -10.0, "solved": null, "grpo": null, "n": 4 }, { "policy": "beam-low", "x": 0.0, "mean": -10.0, "solved": null, "grpo": 0.0, "n": 12 } ], "top_known": false } }, { "suite": "fraud_triage_env", "env": "fraud_triage_env", "notes": "episode is 200 transactions; capped at 15 and scored per step", "episodes": 14, "errors": 1, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 5, "mean": 0.1346666666666667, "std": 0.11174574314537049, "within_task_std": 0.07285393610547089, "grpo_signal": 1.0, "out_tokens": 9350, "invalid_steps": 0, "steps": 75 }, "null": { "episodes": 2, "mean": -0.31333333333333335, "std": 3.925231146709438e-17, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 30 }, "random": { "episodes": 6, "mean": -0.3422222222222222, "std": 0.15499900437752714, "within_task_std": 0.13699590792373983, "grpo_signal": 1.0, "out_tokens": 0, "invalid_steps": 0, "steps": 90 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_mean": -0.31333333333333335, "top": 0.22000000000000006, "gap_fraction": 0.6896551724137933 }, "statistical.reward_reachability": { "pass": true, "reference": "beam-low", "gain_over_floor": 0.44800000000000006, "ci95": [ 0.33333333333333337, 0.5173333333333334 ], "p": 0.0 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 0.44800000000000006, "ci95": [ 0.33333333333333337, 0.5173333333333334 ], "p": 0.0, "n": [ 5, 2 ] }, "statistical.variance_bound": { "pass": true, "within_task_std": 0.07285393610547089, "signal": 0.44800000000000006 }, "statistical.grpo_signal": { "pass": true, "reference": "beam-low", "share": 1.0, "threshold": 0.2 } }, "verdict": "learnable for Beam; not yet tested with a small model", "group": "worth", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 2, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.3103, "mean": -0.31333333333333335, "solved": null, "grpo": null, "n": 2 }, { "policy": "random", "x": 0.273, "mean": -0.3422222222222222, "solved": null, "grpo": 1.0, "n": 6 }, { "policy": "beam-low", "x": 0.8897, "mean": 0.1346666666666667, "solved": null, "grpo": 1.0, "n": 5 } ], "top_known": false } }, { "suite": "grid_world_env", "env": "grid_world_env", "notes": "5x5 grid, reach the goal", "episodes": 14, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": 0.30000000000000004, "std": 0.0, "within_task_std": 0.0, "grpo_signal": 0.0, "out_tokens": 3845, "invalid_steps": 0, "steps": 48 }, "null": { "episodes": 2, "mean": -1.2, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 24 }, "random": { "episodes": 6, "mean": -1.2, "std": 0.0, "within_task_std": 0.0, "grpo_signal": 0.0, "out_tokens": 0, "invalid_steps": 0, "steps": 72 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_mean": -1.2, "top": 0.30000000000000004, "gap_fraction": 1.0 }, "statistical.reward_reachability": { "pass": true, "reference": "beam-low", "gain_over_floor": 1.5, "ci95": [ 1.5, 1.5 ], "p": 0.0 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 1.5, "ci95": [ 1.5, 1.5 ], "p": 0.0, "n": [ 6, 2 ] }, "statistical.variance_bound": { "pass": true, "within_task_std": 0.0, "signal": 1.5 }, "statistical.grpo_signal": { "pass": false, "reference": "beam-low", "share": 0.0, "threshold": 0.2 } }, "verdict": "no gradient for Beam: its repeated attempts at a task almost always agree", "group": "nograd", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 2, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": -1.2, "solved": null, "grpo": null, "n": 2 }, { "policy": "random", "x": 0.0, "mean": -1.2, "solved": null, "grpo": 0.0, "n": 6 }, { "policy": "beam-low", "x": 1.0, "mean": 0.30000000000000004, "solved": null, "grpo": 0.0, "n": 6 } ], "top_known": false } }, { "suite": "hub-Bb-03__os_expert_env", "env": "Bb-03/os_expert_env", "notes": "Diagnose and resolve a production Linux failure in the forensic sandbox.", "episodes": 9, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": 0.2933333333333333, "std": 2.033926143092604, "within_task_std": 0.9166666666666666, "grpo_signal": 0.3333333333333333, "out_tokens": 25213, "invalid_steps": 0, "steps": 90 }, "null": { "episodes": 3, "mean": -0.6233333333333334, "std": 0.05185449728701348, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 45 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_mean": -0.6233333333333334, "top": 4.84, "gap_fraction": 0.9933333333333333 }, "statistical.reward_reachability": { "pass": false, "reference": "beam-low", "gain_over_floor": 0.9166666666666667, "ci95": [ -0.05499999999999994, 2.75 ], "p": 0.16275 }, "statistical.difficulty_separation": { "pass": false, "reference_minus_floor": 0.9166666666666667, "ci95": [ -0.05499999999999994, 2.75 ], "p": 0.17075, "n": [ 6, 3 ] }, "statistical.variance_bound": { "pass": true, "within_task_std": 0.9166666666666666, "signal": 0.9166666666666667 }, "statistical.grpo_signal": { "pass": true, "reference": "beam-low", "share": 0.3333333333333333, "threshold": 0.2 } }, "verdict": "out of reach: Beam scores no better than the floor", "group": "reach", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0067, "mean": -0.6233333333333334, "solved": null, "grpo": null, "n": 3 }, { "policy": "beam-low", "x": 0.1733, "mean": 0.2933333333333333, "solved": null, "grpo": 0.3333333333333333, "n": 6 } ], "top_known": false } }, { "suite": "hub-Boobesh007__code-review", "env": "Boobesh007/code-review", "notes": "Review a code snippet and identify bugs, security vulnerabilities, and quality issues.", "episodes": 9, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": 0.715, "std": 0.15955667749528168, "within_task_std": 0.11499999999999999, "grpo_signal": 1.0, "out_tokens": 1029, "invalid_steps": 0, "steps": 6 }, "null": { "episodes": 3, "mean": 0.3, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 3 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_mean": 0.3, "top": 0.99, "gap_fraction": 1.0 }, "statistical.reward_reachability": { "pass": true, "reference": "beam-low", "gain_over_floor": 0.415, "ci95": [ 0.2916666666666667, 0.5366666666666668 ], "p": 0.0 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 0.415, "ci95": [ 0.2916666666666667, 0.5366666666666668 ], "p": 0.0, "n": [ 6, 3 ] }, "statistical.headroom": { "pass": true, "best_mean": 0.715, "max": 0.99, "room_fraction": 0.3985507246376812 }, "statistical.variance_bound": { "pass": true, "within_task_std": 0.11499999999999999, "signal": 0.415 }, "statistical.grpo_signal": { "pass": true, "reference": "beam-low", "share": 1.0, "threshold": 0.2 } }, "verdict": "learnable for Beam; not yet tested with a small model", "group": "worth", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": 0.3, "solved": null, "grpo": null, "n": 3 }, { "policy": "beam-low", "x": 0.6014, "mean": 0.715, "solved": null, "grpo": 1.0, "n": 6 } ], "top_known": true } }, { "suite": "hub-FineEnvs__geoguesser-env", "env": "FineEnvs/geoguesser-env", "notes": "Determine where the agent was dropped and commit a final coordinate guess.", "episodes": 9, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": 0.0, "std": 0.0, "within_task_std": 0.0, "grpo_signal": 0.0, "out_tokens": 7631, "invalid_steps": 0, "steps": 41 }, "null": { "episodes": 3, "mean": 0.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 45 } }, "checks": { "semantic.floor_gap": { "pass": false, "floor_mean": 0.0, "top": 0.0, "gap_fraction": 0.0 }, "statistical.reward_reachability": { "pass": false, "reference": "beam-low", "gain_over_floor": 0.0, "ci95": [ 0.0, 0.0 ], "p": 1.0 }, "statistical.difficulty_separation": { "pass": false, "reference_minus_floor": 0.0, "ci95": [ 0.0, 0.0 ], "p": 1.0, "n": [ 6, 3 ] }, "statistical.variance_bound": { "pass": true, "within_task_std": 0.0, "signal": 1e-09 }, "statistical.grpo_signal": { "pass": false, "reference": "beam-low", "share": 0.0, "threshold": 0.2 } }, "verdict": "flat: every player tested, Beam included, got the same reward; needs a look", "group": "flat", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": 0.0, "solved": null, "grpo": null, "n": 3 }, { "policy": "beam-low", "x": 0.0, "mean": 0.0, "solved": null, "grpo": 0.0, "n": 6 } ], "top_known": false } }, { "suite": "hub-FineEnvs__nayana-ocr-env", "env": "FineEnvs/nayana-ocr-env", "notes": "Read a page or crop, then transcribe, detect layout, or answer a question about it and receive a score.", "episodes": 9, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": 0.16666666666666666, "std": 0.372677996249965, "within_task_std": 0.16666666666666666, "grpo_signal": 0.3333333333333333, "out_tokens": 1571, "invalid_steps": 0, "steps": 6 }, "null": { "episodes": 3, "mean": 0.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 3 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_mean": 0.0, "top": 1.0, "gap_fraction": 1.0 }, "statistical.reward_reachability": { "pass": false, "reference": "beam-low", "gain_over_floor": 0.16666666666666666, "ci95": [ 0.0, 0.5 ], "p": 0.31775 }, "statistical.difficulty_separation": { "pass": false, "reference_minus_floor": 0.16666666666666666, "ci95": [ 0.0, 0.5 ], "p": 0.34075, "n": [ 6, 3 ] }, "statistical.headroom": { "pass": true, "best_mean": 0.16666666666666666, "max": 1.0, "room_fraction": 0.8333333333333334 }, "statistical.variance_bound": { "pass": false, "within_task_std": 0.16666666666666666, "signal": 0.16666666666666666 }, "statistical.grpo_signal": { "pass": true, "reference": "beam-low", "share": 0.3333333333333333, "threshold": 0.2 } }, "verdict": "out of reach: Beam scores no better than the floor", "group": "reach", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": 0.0, "solved": null, "grpo": null, "n": 3 }, { "policy": "beam-low", "x": 0.1667, "mean": 0.16666666666666666, "solved": null, "grpo": 0.3333333333333333, "n": 6 } ], "top_known": true } }, { "suite": "hub-RavichandraNayakar__customer_support_env", "env": "RavichandraNayakar/customer_support_env", "notes": "Resolve a customer support ticket through the three-phase multi-agent bidding, execution, and resolution protocol.", "episodes": 9, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": 0.0, "std": 0.0, "within_task_std": 0.0, "grpo_signal": 0.0, "out_tokens": 18302, "invalid_steps": 0, "steps": 60, "pass@1": 0.0, "pass@k": 0.0, "k": 2 }, "null": { "episodes": 3, "mean": -0.5, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 3, "pass@1": 0.0, "pass@k": 0.0, "k": 1 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_solve_rate": 0.0, "floor_policy": "null" }, "statistical.reward_reachability": { "pass": false, "reference": "beam-low", "pass@1": 0.0, "pass@k": 0.0, "k": 2 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 0.5, "ci95": [ 0.5, 0.5 ], "p": 0.0, "n": [ 6, 3 ] }, "statistical.headroom": { "pass": true, "best_mean": 0.0, "max": 1.0, "room_fraction": 0.6666666666666666 }, "statistical.variance_bound": { "pass": true, "within_task_std": 0.0, "signal": 0.5 }, "statistical.grpo_signal": { "pass": false, "reference": "beam-low", "share": 0.0, "threshold": 0.2 } }, "verdict": "out of reach: no player tested ever wins, Beam included", "group": "reach", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": -0.5, "solved": 0.0, "grpo": null, "n": 3 }, { "policy": "beam-low", "x": 0.3333, "mean": 0.0, "solved": 0.0, "grpo": 0.0, "n": 6 } ], "top_known": true } }, { "suite": "hub-TejasS1233__openenv-food-delivery-dispatch", "env": "TejasS1233/openenv-food-delivery-dispatch", "notes": "Dispatch couriers to orders and manage deliveries to maximize service quality and efficiency.", "episodes": 9, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": 2.343, "std": 0.0, "within_task_std": 0.0, "grpo_signal": 0.0, "out_tokens": 9073, "invalid_steps": 0, "steps": 90 }, "null": { "episodes": 3, "mean": -8.039999999999997, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 45 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_mean": -8.039999999999997, "top": 2.343, "gap_fraction": 1.0 }, "statistical.reward_reachability": { "pass": true, "reference": "beam-low", "gain_over_floor": 10.382999999999997, "ci95": [ 10.382999999999997, 10.382999999999997 ], "p": 0.0 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 10.382999999999997, "ci95": [ 10.382999999999997, 10.382999999999997 ], "p": 0.0, "n": [ 6, 3 ] }, "statistical.variance_bound": { "pass": true, "within_task_std": 0.0, "signal": 10.382999999999997 }, "statistical.grpo_signal": { "pass": false, "reference": "beam-low", "share": 0.0, "threshold": 0.2 } }, "verdict": "no gradient for Beam: its repeated attempts at a task almost always agree", "declared_max_wrong": { "declared": 1.0, "seen": 2.343 }, "group": "nograd", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": -8.039999999999997, "solved": null, "grpo": null, "n": 3 }, { "policy": "beam-low", "x": 1.0, "mean": 2.343, "solved": null, "grpo": 0.0, "n": 6 } ], "top_known": false } }, { "suite": "hub-Vivaan13__smartinbox-pro", "env": "Vivaan13/smartinbox-pro", "notes": "Classify each incoming email as spam, normal, or urgent.", "episodes": 9, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": 0.3499999999999999, "std": 0.1773649721149397, "within_task_std": 0.05833333333333334, "grpo_signal": 0.6666666666666666, "out_tokens": 8038, "invalid_steps": 0, "steps": 60 }, "null": { "episodes": 3, "mean": 0.34166666666666673, "std": 0.042491829279939886, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 30 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_mean": 0.34166666666666673, "top": 0.55, "gap_fraction": 0.39682539682539675 }, "statistical.reward_reachability": { "pass": false, "reference": "beam-low", "gain_over_floor": 0.008333333333333193, "ci95": [ -0.15, 0.15000000000000008 ], "p": 0.45175 }, "statistical.difficulty_separation": { "pass": false, "reference_minus_floor": 0.008333333333333193, "ci95": [ -0.14583333333333334, 0.15416666666666673 ], "p": 0.44175, "n": [ 6, 3 ] }, "statistical.variance_bound": { "pass": false, "within_task_std": 0.05833333333333334, "signal": 0.008333333333333193 }, "statistical.grpo_signal": { "pass": true, "reference": "beam-low", "share": 0.6666666666666666, "threshold": 0.2 } }, "verdict": "out of reach: Beam scores no better than the floor", "group": "reach", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.6032, "mean": 0.34166666666666673, "solved": null, "grpo": null, "n": 3 }, { "policy": "beam-low", "x": 0.619, "mean": 0.3499999999999999, "solved": null, "grpo": 0.6666666666666666, "n": 6 } ], "top_known": false } }, { "suite": "hub-adishu__auto_jailbreak", "env": "adishu/auto_jailbreak", "notes": "Generate attack prompts that jailbreak the target LLM into revealing protected secrets or producing the forbidden output.", "episodes": 9, "errors": 6, "harness_errors": 0, "tiers": { "null": { "episodes": 3, "mean": 0.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 30 } }, "checks": {}, "verdict": "broken: most episodes error", "group": "broken", "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": 0.0, "solved": null, "grpo": null, "n": 3 } ], "top_known": true } }, { "suite": "hub-anugrahteesdollar__drugenv", "env": "anugrahteesdollar/drugenv", "notes": "Investigate a proposed drug target and submit a calibrated go or no-go validation report.", "episodes": 9, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": 5.697989734708222, "std": 0.402571107206282, "within_task_std": 0.22788196528466273, "grpo_signal": 1.0, "out_tokens": 5425, "invalid_steps": 0, "steps": 41 }, "null": { "episodes": 3, "mean": -1.0629613547546677, "std": 0.0012281489903368888, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 45 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_mean": -1.0629613547546677, "top": 6.288936170212766, "gap_fraction": 0.9998818903682777 }, "statistical.reward_reachability": { "pass": true, "reference": "beam-low", "gain_over_floor": 6.76095108946289, "ci95": [ 6.457033927490856, 7.096578924247419 ], "p": 0.0 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 6.76095108946289, "ci95": [ 6.465177816029766, 7.081847249062861 ], "p": 0.0, "n": [ 6, 3 ] }, "statistical.variance_bound": { "pass": true, "within_task_std": 0.22788196528466273, "signal": 6.76095108946289 }, "statistical.grpo_signal": { "pass": true, "reference": "beam-low", "share": 1.0, "threshold": 0.2 } }, "verdict": "learnable for Beam; not yet tested with a small model", "declared_max_wrong": { "declared": 1.0, "seen": 6.288936170212766 }, "group": "worth", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0001, "mean": -1.0629613547546677, "solved": null, "grpo": null, "n": 3 }, { "policy": "beam-low", "x": 0.9196, "mean": 5.697989734708222, "solved": null, "grpo": 1.0, "n": 6 } ], "top_known": false } }, { "suite": "hub-creovateHQ__BrowserForge", "env": "creovateHQ/BrowserForge", "notes": "Click the no button on the page.", "episodes": 9, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": 10.9, "std": 0.0, "within_task_std": 0.0, "grpo_signal": 0.0, "out_tokens": 394, "invalid_steps": 0, "steps": 6 }, "null": { "episodes": 3, "mean": -2.1, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 18 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_mean": -2.1, "top": 10.9, "gap_fraction": 1.0 }, "statistical.reward_reachability": { "pass": true, "reference": "beam-low", "gain_over_floor": 13.0, "ci95": [ 13.0, 13.0 ], "p": 0.0 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 13.0, "ci95": [ 13.0, 13.0 ], "p": 0.0, "n": [ 6, 3 ] }, "statistical.variance_bound": { "pass": true, "within_task_std": 0.0, "signal": 13.0 }, "statistical.grpo_signal": { "pass": false, "reference": "beam-low", "share": 0.0, "threshold": 0.2 } }, "verdict": "no gradient for Beam: its repeated attempts at a task almost always agree", "group": "nograd", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": -2.1, "solved": null, "grpo": null, "n": 3 }, { "policy": "beam-low", "x": 1.0, "mean": 10.9, "solved": null, "grpo": 0.0, "n": 6 } ], "top_known": false } }, { "suite": "hub-mykrex__webyx-env", "env": "mykrex/webyx-env", "notes": "Audit and fix HTML WCAG accessibility violations in an episode.", "episodes": 9, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": 1.7666666666666668, "std": 0.5436502143433366, "within_task_std": 0.0, "grpo_signal": 0.0, "out_tokens": 4061, "invalid_steps": 0, "steps": 32 }, "null": { "episodes": 3, "mean": -0.7999999999999999, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 24 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_mean": -0.7999999999999999, "top": 2.5000000000000004, "gap_fraction": 1.0 }, "statistical.reward_reachability": { "pass": true, "reference": "beam-low", "gain_over_floor": 2.566666666666667, "ci95": [ 2.1333333333333333, 3.0 ], "p": 0.0 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 2.566666666666667, "ci95": [ 2.1333333333333333, 3.0 ], "p": 0.0, "n": [ 6, 3 ] }, "statistical.variance_bound": { "pass": true, "within_task_std": 0.0, "signal": 2.566666666666667 }, "statistical.grpo_signal": { "pass": false, "reference": "beam-low", "share": 0.0, "threshold": 0.2 } }, "verdict": "no gradient for Beam: its repeated attempts at a task almost always agree", "declared_max_wrong": { "declared": 1.0, "seen": 2.5000000000000004 }, "group": "nograd", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": -0.7999999999999999, "solved": null, "grpo": null, "n": 3 }, { "policy": "beam-low", "x": 0.7778, "mean": 1.7666666666666668, "solved": null, "grpo": 0.0, "n": 6 } ], "top_known": false } }, { "suite": "hub-openenv__atari_env", "env": "openenv/atari_env", "notes": "Play an Atari 2600 game by choosing legal actions to maximize the cumulative episode reward.", "episodes": 18, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": 0.0, "std": 0.0, "within_task_std": 0.0, "grpo_signal": 0.0, "out_tokens": 23059, "invalid_steps": 0, "steps": 90 }, "null": { "episodes": 3, "mean": 0.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 45 }, "random": { "episodes": 9, "mean": 0.0, "std": 0.0, "within_task_std": 0.0, "grpo_signal": 0.0, "out_tokens": 0, "invalid_steps": 0, "steps": 135 } }, "checks": { "semantic.floor_gap": { "pass": false, "floor_mean": 0.0, "top": 0.0, "gap_fraction": 0.0 }, "statistical.reward_reachability": { "pass": false, "reference": "beam-low", "gain_over_floor": 0.0, "ci95": [ 0.0, 0.0 ], "p": 1.0 }, "statistical.difficulty_separation": { "pass": false, "reference_minus_floor": 0.0, "ci95": [ 0.0, 0.0 ], "p": 1.0, "n": [ 6, 3 ] }, "statistical.variance_bound": { "pass": true, "within_task_std": 0.0, "signal": 1e-09 }, "statistical.grpo_signal": { "pass": false, "reference": "beam-low", "share": 0.0, "threshold": 0.2 } }, "verdict": "flat: every player tested, Beam included, got the same reward; needs a look", "group": "flat", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": 0.0, "solved": null, "grpo": null, "n": 3 }, { "policy": "random", "x": 0.0, "mean": 0.0, "solved": null, "grpo": 0.0, "n": 9 }, { "policy": "beam-low", "x": 0.0, "mean": 0.0, "solved": null, "grpo": 0.0, "n": 6 } ], "top_known": false } }, { "suite": "hub-openenv__coding_env", "env": "openenv/coding_env", "notes": "Execute Python code and inspect its output.", "episodes": 9, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": -0.09999999999999994, "std": 1.2096831541082702, "within_task_std": 0.5333333333333333, "grpo_signal": 0.6666666666666666, "out_tokens": 6007, "invalid_steps": 0, "steps": 30 }, "null": { "episodes": 3, "mean": 0.5, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 15 } }, "checks": { "semantic.floor_gap": { "pass": false, "floor_mean": 0.5, "top": 0.5, "gap_fraction": 0.0 }, "statistical.reward_reachability": { "pass": false, "reference": "beam-low", "gain_over_floor": -0.6, "ci95": [ -1.6833333333333333, -0.016666666666666663 ], "p": 1.0 }, "statistical.difficulty_separation": { "pass": false, "reference_minus_floor": -0.6, "ci95": [ -1.6833333333333333, -0.016666666666666663 ], "p": 1.0, "n": [ 6, 3 ] }, "statistical.variance_bound": { "pass": false, "within_task_std": 0.5333333333333333, "signal": 1e-09 }, "statistical.grpo_signal": { "pass": true, "reference": "beam-low", "share": 0.6666666666666666, "threshold": 0.2 } }, "verdict": "weak signal: doing nothing already scores near the top", "group": "weak", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 1.0, "mean": 0.5, "solved": null, "grpo": null, "n": 3 }, { "policy": "beam-low", "x": 0.8182, "mean": -0.09999999999999994, "solved": null, "grpo": 0.6666666666666666, "n": 6 } ], "top_known": false } }, { "suite": "hub-openenv__echo_env", "env": "openenv/echo_env", "notes": "Call echo tools to echo messages back from the environment.", "episodes": 9, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": 0.0, "std": 0.0, "within_task_std": 0.0, "grpo_signal": 0.0, "out_tokens": 3531, "invalid_steps": 0, "steps": 18 }, "null": { "episodes": 3, "mean": 0.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 9 } }, "checks": { "semantic.floor_gap": { "pass": false, "floor_mean": 0.0, "top": 0.0, "gap_fraction": 0.0 }, "statistical.reward_reachability": { "pass": false, "reference": "beam-low", "gain_over_floor": 0.0, "ci95": [ 0.0, 0.0 ], "p": 1.0 }, "statistical.difficulty_separation": { "pass": false, "reference_minus_floor": 0.0, "ci95": [ 0.0, 0.0 ], "p": 1.0, "n": [ 6, 3 ] }, "statistical.headroom": { "pass": false, "best_mean": 0.0, "max": 0.0, "room_fraction": 0.0 }, "statistical.variance_bound": { "pass": true, "within_task_std": 0.0, "signal": 1e-09 }, "statistical.grpo_signal": { "pass": false, "reference": "beam-low", "share": 0.0, "threshold": 0.2 } }, "verdict": "flat: every player tested, Beam included, got the same reward; needs a look", "group": "flat", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": 0.0, "solved": null, "grpo": null, "n": 3 }, { "policy": "beam-low", "x": 0.0, "mean": 0.0, "solved": null, "grpo": 0.0, "n": 6 } ], "top_known": false } }, { "suite": "hub-openenv__openspiel_env", "env": "openenv/openspiel_env", "notes": "Move horizontally to catch the falling ball.", "episodes": 18, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": 0.0, "std": 1.0, "within_task_std": 0.3333333333333333, "grpo_signal": 0.3333333333333333, "out_tokens": 31019, "invalid_steps": 0, "steps": 54 }, "null": { "episodes": 3, "mean": -1.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 27 }, "random": { "episodes": 9, "mean": -0.3333333333333333, "std": 0.9428090415820634, "within_task_std": 0.6285393610547089, "grpo_signal": 0.6666666666666666, "out_tokens": 0, "invalid_steps": 0, "steps": 81 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_mean": -0.3333333333333333, "top": 1.0, "gap_fraction": 0.6666666666666666 }, "statistical.reward_reachability": { "pass": false, "reference": "beam-low", "gain_over_floor": 0.3333333333333333, "ci95": [ -0.6666666666666666, 1.3333333333333333 ], "p": 0.3025 }, "statistical.difficulty_separation": { "pass": false, "reference_minus_floor": 0.3333333333333333, "ci95": [ -0.6666666666666666, 1.3333333333333333 ], "p": 0.30525, "n": [ 6, 9 ] }, "statistical.headroom": { "pass": true, "best_mean": 0.0, "max": 1.0, "room_fraction": 0.5 }, "statistical.variance_bound": { "pass": false, "within_task_std": 0.3333333333333333, "signal": 0.3333333333333333 }, "statistical.grpo_signal": { "pass": true, "reference": "beam-low", "share": 0.3333333333333333, "threshold": 0.2 } }, "verdict": "out of reach: Beam scores no better than the floor", "group": "reach", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": -1.0, "solved": null, "grpo": null, "n": 3 }, { "policy": "random", "x": 0.3333, "mean": -0.3333333333333333, "solved": null, "grpo": 0.6666666666666666, "n": 9 }, { "policy": "beam-low", "x": 0.5, "mean": 0.0, "solved": null, "grpo": 0.3333333333333333, "n": 6 } ], "top_known": true } }, { "suite": "hub-sergiopaniego__repl", "env": "sergiopaniego/repl", "notes": "Execute Python code iteratively in a sandboxed REPL to analyze a provided context and produce a final answer.", "episodes": 9, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": 0.725, "std": 0.16266017746619277, "within_task_std": 0.075, "grpo_signal": 0.6666666666666666, "out_tokens": 7972, "invalid_steps": 0, "steps": 51 }, "null": { "episodes": 3, "mean": 0.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 45 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_mean": 0.0, "top": 1.0, "gap_fraction": 1.0 }, "statistical.reward_reachability": { "pass": true, "reference": "beam-low", "gain_over_floor": 0.725, "ci95": [ 0.5916666666666668, 0.8583333333333334 ], "p": 0.0 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 0.725, "ci95": [ 0.6, 0.8583333333333334 ], "p": 0.0, "n": [ 6, 3 ] }, "statistical.headroom": { "pass": true, "best_mean": 0.725, "max": 1.0, "room_fraction": 0.275 }, "statistical.variance_bound": { "pass": true, "within_task_std": 0.075, "signal": 0.725 }, "statistical.grpo_signal": { "pass": true, "reference": "beam-low", "share": 0.6666666666666666, "threshold": 0.2 } }, "verdict": "learnable for Beam; not yet tested with a small model", "group": "worth", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": 0.0, "solved": null, "grpo": null, "n": 3 }, { "policy": "beam-low", "x": 0.725, "mean": 0.725, "solved": null, "grpo": 0.6666666666666666, "n": 6 } ], "top_known": true } }, { "suite": "hub-sergiopaniego__wordle", "env": "sergiopaniego/wordle", "notes": "Guess the secret 5-letter Wordle word within 6 attempts.", "episodes": 9, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": 0.19444444444444442, "std": 0.0921284663987611, "within_task_std": 0.05555555555555556, "grpo_signal": 0.6666666666666666, "out_tokens": 14009, "invalid_steps": 0, "steps": 28 }, "null": { "episodes": 3, "mean": 0.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 6 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_mean": 0.0, "top": 0.25, "gap_fraction": 1.0 }, "statistical.reward_reachability": { "pass": true, "reference": "beam-low", "gain_over_floor": 0.19444444444444442, "ci95": [ 0.1111111111111111, 0.25 ], "p": 0.0 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 0.19444444444444442, "ci95": [ 0.1111111111111111, 0.25 ], "p": 0.0, "n": [ 6, 3 ] }, "statistical.variance_bound": { "pass": true, "within_task_std": 0.05555555555555556, "signal": 0.19444444444444442 }, "statistical.grpo_signal": { "pass": true, "reference": "beam-low", "share": 0.6666666666666666, "threshold": 0.2 } }, "verdict": "learnable for Beam; not yet tested with a small model", "group": "worth", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": 0.0, "solved": null, "grpo": null, "n": 3 }, { "policy": "beam-low", "x": 0.7778, "mean": 0.19444444444444442, "solved": null, "grpo": 0.6666666666666666, "n": 6 } ], "top_known": false } }, { "suite": "hub-souller__email-triage-env", "env": "souller/email-triage-env", "notes": "Triage emails by classifying, summarising, routing, and replying according to the selected task difficulty.", "episodes": 9, "errors": 3, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 3, "mean": 0.6188888888888889, "std": 0.23414673040216621, "within_task_std": null, "grpo_signal": null, "out_tokens": 1528, "invalid_steps": 0, "steps": 6 }, "null": { "episodes": 3, "mean": 0.01, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 6 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_mean": 0.01, "top": 0.99, "gap_fraction": 1.0 }, "statistical.reward_reachability": { "pass": true, "reference": "beam-low", "gain_over_floor": 0.6088888888888889, "ci95": [ 0.32666666666666666, 0.9 ], "p": 0.0 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 0.6088888888888889, "ci95": [ 0.32666666666666666, 0.9 ], "p": 0.0, "n": [ 3, 3 ] }, "statistical.headroom": { "pass": true, "best_mean": 0.6188888888888889, "max": 0.99, "room_fraction": 0.37868480725623577 } }, "verdict": "learnable for Beam; not yet tested with a small model", "group": "worth", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": 0.01, "solved": null, "grpo": null, "n": 3 }, { "policy": "beam-low", "x": 0.6213, "mean": 0.6188888888888889, "solved": null, "grpo": null, "n": 3 } ], "top_known": true } }, { "suite": "hub-yaswanth169__APISHIFT", "env": "yaswanth169/APISHIFT", "notes": "Migrate the client code to safely handle the breaking API changes in the scenario and submit a final migration plan.", "episodes": 9, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": 0.01, "std": 0.0, "within_task_std": 0.0, "grpo_signal": 0.0, "out_tokens": 8493, "invalid_steps": 0, "steps": 36 }, "null": { "episodes": 3, "mean": 0.01, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 18 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_mean": 0.01, "top": 0.99, "gap_fraction": 1.0 }, "statistical.reward_reachability": { "pass": false, "reference": "beam-low", "gain_over_floor": 0.0, "ci95": [ 0.0, 0.0 ], "p": 1.0 }, "statistical.difficulty_separation": { "pass": false, "reference_minus_floor": 0.0, "ci95": [ 0.0, 0.0 ], "p": 1.0, "n": [ 6, 3 ] }, "statistical.headroom": { "pass": true, "best_mean": 0.01, "max": 0.99, "room_fraction": 1.0 }, "statistical.variance_bound": { "pass": true, "within_task_std": 0.0, "signal": 1e-09 }, "statistical.grpo_signal": { "pass": false, "reference": "beam-low", "share": 0.0, "threshold": 0.2 } }, "verdict": "flat: every player tested, Beam included, got the same reward; needs a look", "group": "flat", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": 0.01, "solved": null, "grpo": null, "n": 3 }, { "policy": "beam-low", "x": 0.0, "mean": 0.01, "solved": null, "grpo": 0.0, "n": 6 } ], "top_known": true } }, { "suite": "hub-yaswanth169__data-janitor-env", "env": "yaswanth169/data-janitor-env", "notes": "Clean a messy tabular dataset by applying transformations so it matches the target schema and ground truth, then submit.", "episodes": 9, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": 0.29466666666666663, "std": 0.44212045744218725, "within_task_std": 0.3377666666666667, "grpo_signal": 1.0, "out_tokens": 14193, "invalid_steps": 0, "steps": 55 }, "null": { "episodes": 3, "mean": 0.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 30 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_mean": 0.0, "top": 1.0535, "gap_fraction": 0.742476566354218 }, "statistical.reward_reachability": { "pass": true, "reference": "beam-low", "gain_over_floor": 0.29466666666666663, "ci95": [ -0.04310000000000003, 0.6702833333333333 ], "p": 0.04425 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 0.29466666666666663, "ci95": [ -0.044050000000000034, 0.6519333333333334 ], "p": 0.04725, "n": [ 6, 3 ] }, "statistical.variance_bound": { "pass": false, "within_task_std": 0.3377666666666667, "signal": 0.29466666666666663 }, "statistical.grpo_signal": { "pass": true, "reference": "beam-low", "share": 1.0, "threshold": 0.2 } }, "verdict": "learnable for Beam; not yet tested with a small model", "declared_max_wrong": { "declared": 1.0, "seen": 1.0535 }, "group": "worth", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.2575, "mean": 0.0, "solved": null, "grpo": null, "n": 3 }, { "policy": "beam-low", "x": 0.4652, "mean": 0.29466666666666663, "solved": null, "grpo": 1.0, "n": 6 } ], "top_known": false } }, { "suite": "maze_env", "env": "maze_env", "notes": "8x8 maze; +10 at the exit, small penalties per move; capped at 25 moves", "episodes": 24, "errors": 0, "harness_errors": 0, "tiers": { "beam-high": { "episodes": 3, "mean": 9.35, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 79449, "invalid_steps": 0, "steps": 42, "pass@1": 1.0, "pass@k": 1.0, "k": 1 }, "beam-low": { "episodes": 9, "mean": 9.283333333333335, "std": 0.094280904158206, "within_task_std": 0.06285393610547067, "grpo_signal": 0.6666666666666666, "out_tokens": 34180, "invalid_steps": 0, "steps": 138, "pass@1": 1.0, "pass@k": 1.0, "k": 3 }, "null": { "episodes": 3, "mean": -5.849999999999999, "std": 8.881784197001252e-16, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 75, "pass@1": 0.0, "pass@k": 0.0, "k": 1 }, "random": { "episodes": 9, "mean": -4.538888888888889, "std": 0.40123266856150647, "within_task_std": 0.3864308036286846, "grpo_signal": 1.0, "out_tokens": 0, "invalid_steps": 0, "steps": 225, "pass@1": 0.0, "pass@k": 0.0, "k": 3 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_solve_rate": 0.0, "floor_policy": "random" }, "statistical.reward_reachability": { "pass": true, "reference": "beam-high", "pass@1": 1.0, "pass@k": 1.0, "k": 1 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 13.88888888888889, "ci95": [ 13.622222222222222, 14.155555555555555 ], "p": 0.0, "n": [ 3, 9 ], "between_beam_efforts": { "stronger": "beam-high", "weaker": "beam-low", "diff": 0.06666666666666465, "ci95": [ 0.0, 0.13333333333333286 ], "p": 0.02825, "separates": true } }, "statistical.headroom": { "pass": false, "best_mean": 9.35, "max": 10.0, "room_fraction": 0.0410094637223975 } }, "verdict": "too easy for Beam; not yet tested with a small model", "group": "easy", "reference": "beam-high", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": -5.849999999999999, "solved": 0.0, "grpo": null, "n": 3 }, { "policy": "random", "x": 0.0827, "mean": -4.538888888888889, "solved": 0.0, "grpo": 1.0, "n": 9 }, { "policy": "beam-low", "x": 0.9548, "mean": 9.283333333333335, "solved": 1.0, "grpo": 0.6666666666666666, "n": 9 }, { "policy": "beam-high", "x": 0.959, "mean": 9.35, "solved": 1.0, "grpo": null, "n": 3 } ], "top_known": true } }, { "suite": "os-2048", "env": "openspiel_env", "notes": "OpenSpiel 2048 (server default opponent)", "episodes": 14, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": 60.0, "std": 14.787382008545888, "within_task_std": 13.412614930262478, "grpo_signal": 1.0, "out_tokens": 76422, "invalid_steps": 0, "steps": 90 }, "null": { "episodes": 2, "mean": 64.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 30 }, "random": { "episodes": 6, "mean": 54.666666666666664, "std": 10.498677165349081, "within_task_std": 6.599663291074444, "grpo_signal": 1.0, "out_tokens": 0, "invalid_steps": 0, "steps": 90 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_mean": 64.0, "top": 84.0, "gap_fraction": 0.45454545454545453 }, "statistical.reward_reachability": { "pass": false, "reference": "beam-low", "gain_over_floor": -4.0, "ci95": [ -15.333333333333336, 7.350000000000056 ], "p": 0.762 }, "statistical.difficulty_separation": { "pass": false, "reference_minus_floor": -4.0, "ci95": [ -15.333333333333336, 7.333333333333329 ], "p": 0.76525, "n": [ 6, 2 ] }, "statistical.variance_bound": { "pass": false, "within_task_std": 13.412614930262478, "signal": 1e-09 }, "statistical.grpo_signal": { "pass": true, "reference": "beam-low", "share": 1.0, "threshold": 0.2 } }, "verdict": "out of reach: Beam scores no better than the floor", "group": "reach", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 2, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.5455, "mean": 64.0, "solved": null, "grpo": null, "n": 2 }, { "policy": "random", "x": 0.3333, "mean": 54.666666666666664, "solved": null, "grpo": 1.0, "n": 6 }, { "policy": "beam-low", "x": 0.4545, "mean": 60.0, "solved": null, "grpo": 1.0, "n": 6 } ], "top_known": false } }, { "suite": "os-blackjack", "env": "openspiel_env", "notes": "OpenSpiel blackjack (server default opponent)", "episodes": 21, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 9, "mean": -0.5555555555555556, "std": 0.8314794192830981, "within_task_std": 0.6285393610547089, "grpo_signal": 0.6666666666666666, "out_tokens": 4224, "invalid_steps": 0, "steps": 10, "pass@1": 0.2222222222222222, "pass@k": 0.6666666666666666, "k": 3 }, "null": { "episodes": 3, "mean": -1.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 8, "pass@1": 0.0, "pass@k": 0.0, "k": 1 }, "random": { "episodes": 9, "mean": -0.1111111111111111, "std": 0.9938079899999066, "within_task_std": 0.31426968052735443, "grpo_signal": 0.3333333333333333, "out_tokens": 0, "invalid_steps": 0, "steps": 15, "pass@1": 0.4444444444444444, "pass@k": 0.6666666666666666, "k": 3 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_solve_rate": 0.4444444444444444, "floor_policy": "random" }, "statistical.reward_reachability": { "pass": true, "reference": "beam-low", "pass@1": 0.2222222222222222, "pass@k": 0.6666666666666666, "k": 3 }, "statistical.difficulty_separation": { "pass": false, "reference_minus_floor": -0.4444444444444445, "ci95": [ -1.3333333333333333, 0.4444444444444445 ], "p": 0.8975, "n": [ 9, 9 ] }, "statistical.headroom": { "pass": true, "best_mean": -0.5555555555555556, "max": 1.0, "room_fraction": 0.7777777777777778 }, "statistical.variance_bound": { "pass": false, "within_task_std": 0.6285393610547089, "signal": 1e-09 }, "statistical.grpo_signal": { "pass": true, "reference": "beam-low", "share": 0.6666666666666666, "threshold": 0.2 } }, "verdict": "skill doesn't show yet: Beam isn't clearly above the floor in these games", "group": "unsure", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": -1.0, "solved": 0.0, "grpo": null, "n": 3 }, { "policy": "random", "x": 0.4444, "mean": -0.1111111111111111, "solved": 0.4444444444444444, "grpo": 0.3333333333333333, "n": 9 }, { "policy": "beam-low", "x": 0.2222, "mean": -0.5555555555555556, "solved": 0.2222222222222222, "grpo": 0.6666666666666666, "n": 9 } ], "top_known": true } }, { "suite": "os-catch", "env": "openspiel_env", "notes": "OpenSpiel catch (server default opponent)", "episodes": 21, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 9, "mean": -0.7777777777777778, "std": 0.6285393610547089, "within_task_std": 0.31426968052735443, "grpo_signal": 0.3333333333333333, "out_tokens": 43737, "invalid_steps": 0, "steps": 81, "pass@1": 0.1111111111111111, "pass@k": 0.3333333333333333, "k": 3 }, "null": { "episodes": 3, "mean": -0.3333333333333333, "std": 0.9428090415820634, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 27, "pass@1": 0.3333333333333333, "pass@k": 0.3333333333333333, "k": 1 }, "random": { "episodes": 9, "mean": -0.7777777777777778, "std": 0.6285393610547089, "within_task_std": 0.31426968052735443, "grpo_signal": 0.3333333333333333, "out_tokens": 0, "invalid_steps": 0, "steps": 81, "pass@1": 0.1111111111111111, "pass@k": 0.3333333333333333, "k": 3 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_solve_rate": 0.3333333333333333, "floor_policy": "null" }, "statistical.reward_reachability": { "pass": true, "reference": "beam-low", "pass@1": 0.1111111111111111, "pass@k": 0.3333333333333333, "k": 3 }, "statistical.difficulty_separation": { "pass": false, "reference_minus_floor": -0.4444444444444445, "ci95": [ -1.7777777777777777, 0.4444444444444444 ], "p": 0.8025, "n": [ 9, 3 ] }, "statistical.headroom": { "pass": true, "best_mean": -0.7777777777777778, "max": 1.0, "room_fraction": 0.8888888888888888 }, "statistical.variance_bound": { "pass": false, "within_task_std": 0.31426968052735443, "signal": 1e-09 }, "statistical.grpo_signal": { "pass": true, "reference": "beam-low", "share": 0.3333333333333333, "threshold": 0.2 } }, "verdict": "skill doesn't show yet: Beam isn't clearly above the floor in these games", "group": "unsure", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.3333, "mean": -0.3333333333333333, "solved": 0.3333333333333333, "grpo": null, "n": 3 }, { "policy": "random", "x": 0.1111, "mean": -0.7777777777777778, "solved": 0.1111111111111111, "grpo": 0.3333333333333333, "n": 9 }, { "policy": "beam-low", "x": 0.1111, "mean": -0.7777777777777778, "solved": 0.1111111111111111, "grpo": 0.3333333333333333, "n": 9 } ], "top_known": true } }, { "suite": "os-cliff_walking", "env": "openspiel_env", "notes": "OpenSpiel cliff_walking (server default opponent)", "episodes": 14, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": -100.5, "std": 1.118033988749895, "within_task_std": 0.7071067811865476, "grpo_signal": 0.5, "out_tokens": 1137, "invalid_steps": 0, "steps": 9 }, "null": { "episodes": 2, "mean": -100.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 2 }, "random": { "episodes": 6, "mean": -104.33333333333333, "std": 3.2998316455372216, "within_task_std": 2.250217458768164, "grpo_signal": 1.0, "out_tokens": 0, "invalid_steps": 0, "steps": 32 } }, "checks": { "semantic.floor_gap": { "pass": false, "floor_mean": -100.0, "top": -100.0, "gap_fraction": 0.0 }, "statistical.reward_reachability": { "pass": false, "reference": "beam-low", "gain_over_floor": -0.5, "ci95": [ -1.5, 0.0 ], "p": 1.0 }, "statistical.difficulty_separation": { "pass": false, "reference_minus_floor": -0.5, "ci95": [ -1.5, 0.0 ], "p": 1.0, "n": [ 6, 2 ] }, "statistical.variance_bound": { "pass": false, "within_task_std": 0.7071067811865476, "signal": 1e-09 }, "statistical.grpo_signal": { "pass": true, "reference": "beam-low", "share": 0.5, "threshold": 0.2 } }, "verdict": "weak signal: doing nothing already scores near the top", "group": "weak", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 2, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 1.0, "mean": -100.0, "solved": null, "grpo": null, "n": 2 }, { "policy": "random", "x": 0.5667, "mean": -104.33333333333333, "solved": null, "grpo": 1.0, "n": 6 }, { "policy": "beam-low", "x": 0.95, "mean": -100.5, "solved": null, "grpo": 0.5, "n": 6 } ], "top_known": false } }, { "suite": "os-kuhn_poker", "env": "openspiel_env", "notes": "OpenSpiel kuhn_poker (server default opponent)", "episodes": 21, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 9, "mean": 0.7777777777777778, "std": 1.0304020550550783, "within_task_std": 0.7856742013183862, "grpo_signal": 1.0, "out_tokens": 5531, "invalid_steps": 0, "steps": 11 }, "null": { "episodes": 3, "mean": 0.3333333333333333, "std": 0.9428090415820634, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 4 }, "random": { "episodes": 9, "mean": -0.5555555555555556, "std": 1.4229164972072998, "within_task_std": 1.0999438818457408, "grpo_signal": 1.0, "out_tokens": 0, "invalid_steps": 0, "steps": 13 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_mean": 0.3333333333333333, "top": 2.0, "gap_fraction": 0.4166666666666667 }, "statistical.reward_reachability": { "pass": false, "reference": "beam-low", "gain_over_floor": 0.4444444444444445, "ci95": [ -0.7777777777777778, 1.7777777777777777 ], "p": 0.3025 }, "statistical.difficulty_separation": { "pass": false, "reference_minus_floor": 0.4444444444444445, "ci95": [ -0.7777777777777778, 1.7777777777777777 ], "p": 0.29075, "n": [ 9, 3 ] }, "statistical.variance_bound": { "pass": false, "within_task_std": 0.7856742013183862, "signal": 0.4444444444444445 }, "statistical.grpo_signal": { "pass": true, "reference": "beam-low", "share": 1.0, "threshold": 0.2 } }, "verdict": "out of reach: Beam scores no better than the floor", "group": "reach", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.5833, "mean": 0.3333333333333333, "solved": null, "grpo": null, "n": 3 }, { "policy": "random", "x": 0.3611, "mean": -0.5555555555555556, "solved": null, "grpo": 1.0, "n": 9 }, { "policy": "beam-low", "x": 0.6944, "mean": 0.7777777777777778, "solved": null, "grpo": 1.0, "n": 9 } ], "top_known": false } }, { "suite": "os-tic_tac_toe", "env": "openspiel_env", "notes": "OpenSpiel tic_tac_toe (server default opponent)", "episodes": 24, "errors": 0, "harness_errors": 0, "tiers": { "beam-high": { "episodes": 3, "mean": 1.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 39268, "invalid_steps": 0, "steps": 9, "pass@1": 1.0, "pass@k": 1.0, "k": 1 }, "beam-low": { "episodes": 9, "mean": 0.8888888888888888, "std": 0.3142696805273545, "within_task_std": 0.15713484026367722, "grpo_signal": 0.3333333333333333, "out_tokens": 22468, "invalid_steps": 0, "steps": 35, "pass@1": 0.8888888888888888, "pass@k": 1.0, "k": 3 }, "null": { "episodes": 3, "mean": 1.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 9, "pass@1": 1.0, "pass@k": 1.0, "k": 1 }, "random": { "episodes": 9, "mean": 0.3333333333333333, "std": 0.816496580927726, "within_task_std": 0.4714045207910316, "grpo_signal": 0.6666666666666666, "out_tokens": 0, "invalid_steps": 0, "steps": 39, "pass@1": 0.5555555555555556, "pass@k": 1.0, "k": 3 } }, "checks": { "semantic.floor_gap": { "pass": false, "floor_solve_rate": 1.0, "floor_policy": "null" }, "statistical.reward_reachability": { "pass": true, "reference": "beam-high", "pass@1": 1.0, "pass@k": 1.0, "k": 1 }, "statistical.difficulty_separation": { "pass": false, "reference_minus_floor": 0.0, "ci95": [ 0.0, 0.0 ], "p": 1.0, "n": [ 3, 3 ], "between_beam_efforts": { "stronger": "beam-high", "weaker": "beam-low", "diff": 0.11111111111111116, "ci95": [ 0.0, 0.33333333333333337 ], "p": 0.341, "separates": false } }, "statistical.headroom": { "pass": false, "best_mean": 1.0, "max": 1.0, "room_fraction": 0.0 } }, "verdict": "weak signal: a do-nothing or random policy already solves 100%", "group": "weak", "reference": "beam-high", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 1.0, "mean": 1.0, "solved": 1.0, "grpo": null, "n": 3 }, { "policy": "random", "x": 0.6667, "mean": 0.3333333333333333, "solved": 0.5555555555555556, "grpo": 0.6666666666666666, "n": 9 }, { "policy": "beam-low", "x": 0.9444, "mean": 0.8888888888888888, "solved": 0.8888888888888888, "grpo": 0.3333333333333333, "n": 9 }, { "policy": "beam-high", "x": 1.0, "mean": 1.0, "solved": 1.0, "grpo": null, "n": 3 } ], "top_known": true } }, { "suite": "pelican_svg_env", "env": "pelican_svg_env", "notes": "draw an SVG from a text prompt; layered score", "episodes": 12, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 9, "mean": 1.0, "std": 0.0, "within_task_std": 0.0, "grpo_signal": 0.0, "out_tokens": 11376, "invalid_steps": 0, "steps": 9 }, "null": { "episodes": 3, "mean": 0.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 3 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_mean": 0.0, "top": 1.0, "gap_fraction": 1.0 }, "statistical.reward_reachability": { "pass": true, "reference": "beam-low", "gain_over_floor": 1.0, "ci95": [ 1.0, 1.0 ], "p": 0.0 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 1.0, "ci95": [ 1.0, 1.0 ], "p": 0.0, "n": [ 9, 3 ] }, "statistical.headroom": { "pass": false, "best_mean": 1.0, "max": 1.0, "room_fraction": 0.0 }, "statistical.variance_bound": { "pass": true, "within_task_std": 0.0, "signal": 1.0 }, "statistical.grpo_signal": { "pass": false, "reference": "beam-low", "share": 0.0, "threshold": 0.2 } }, "verdict": "broken as shipped: the vision judge is off by default, so the reward checks only shapes and any two-wheeled drawing scores 1.00", "defect": "The vision judge is off by default, so the reward checks only shapes and any two-wheeled drawing scores 1.00.", "group": "broken", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 3, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": 0.0, "solved": null, "grpo": null, "n": 3 }, { "policy": "beam-low", "x": 1.0, "mean": 1.0, "solved": null, "grpo": 0.0, "n": 9 } ], "top_known": true } }, { "suite": "rg-basic_arithmetic", "env": "reasoning_gym_env", "notes": "reasoning_gym family basic_arithmetic", "episodes": 64, "errors": 0, "harness_errors": 0, "tiers": { "beam-high": { "episodes": 8, "mean": 0.875, "std": 0.33071891388307384, "within_task_std": null, "grpo_signal": null, "out_tokens": 1674, "invalid_steps": 0, "steps": 8, "pass@1": 0.875, "pass@k": 0.875, "k": 1 }, "beam-low": { "episodes": 24, "mean": 0.875, "std": 0.33071891388307384, "within_task_std": 0.0, "grpo_signal": 0.0, "out_tokens": 4075, "invalid_steps": 0, "steps": 24, "pass@1": 0.875, "pass@k": 0.875, "k": 3 }, "null": { "episodes": 8, "mean": 0.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 8, "pass@1": 0.0, "pass@k": 0.0, "k": 1 }, "small": { "episodes": 24, "mean": 0.03888375600689673, "std": 0.06005724529505568, "within_task_std": 0.02657878796645543, "grpo_signal": 0.5, "out_tokens": 983, "invalid_steps": 0, "steps": 24, "pass@1": 0.0, "pass@k": 0.0, "k": 3 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_solve_rate": 0.0, "floor_policy": "null" }, "statistical.reward_reachability": { "pass": true, "reference": "beam-high", "pass@1": 0.875, "pass@k": 0.875, "k": 1 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 0.875, "ci95": [ 0.625, 1.0 ], "p": 0.0, "n": [ 8, 8 ], "between_beam_efforts": { "stronger": "beam-high", "weaker": "beam-high", "diff": 0.0, "ci95": [ -0.375, 0.375 ], "p": 0.655, "separates": false } }, "statistical.headroom": { "pass": true, "best_mean": 0.875, "max": 1.0, "room_fraction": 0.125 }, "trainee.reachability": { "pass": false, "pass@1": 0.0, "pass@k": 0.0, "k": 3 }, "trainee.grpo_signal": { "pass": true, "share": 0.5, "threshold": 0.2 }, "trainee.rule": { "prediction": "improves" }, "trainee.room_below_reference": { "pass": true, "reference_minus_trainee": 0.8361162439931032, "p": 0.0 }, "statistical.adversarial_floor": { "pass": true, "strategy": "constant", "label": "always \"-2\"", "answer": "-2", "score": 0.013, "learner_mean": 0.039, "added_after_training": true } }, "verdict": "worth training the small model on", "group": "worth", "reference": "beam-high", "trainee": "small", "judged_for": "small model", "tasks": 8, "provisional": false, "gameable": false, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": 0.0, "solved": 0.0, "grpo": null, "n": 8 }, { "policy": "small", "x": 0.0389, "mean": 0.03888375600689673, "solved": 0.0, "grpo": 0.5, "n": 24 }, { "policy": "beam-low", "x": 0.875, "mean": 0.875, "solved": 0.875, "grpo": 0.0, "n": 24 }, { "policy": "beam-high", "x": 0.875, "mean": 0.875, "solved": 0.875, "grpo": null, "n": 8 } ], "top_known": true } }, { "suite": "rg-chain_sum", "env": "reasoning_gym_env", "notes": "reasoning_gym family chain_sum", "episodes": 64, "errors": 0, "harness_errors": 0, "tiers": { "beam-high": { "episodes": 8, "mean": 1.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 622, "invalid_steps": 0, "steps": 8, "pass@1": 1.0, "pass@k": 1.0, "k": 1 }, "beam-low": { "episodes": 24, "mean": 1.0, "std": 0.0, "within_task_std": 0.0, "grpo_signal": 0.0, "out_tokens": 1749, "invalid_steps": 0, "steps": 24, "pass@1": 1.0, "pass@k": 1.0, "k": 3 }, "null": { "episodes": 8, "mean": 0.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 8, "pass@1": 0.0, "pass@k": 0.0, "k": 1 }, "small": { "episodes": 24, "mean": 0.028928034934775193, "std": 0.0486549908230185, "within_task_std": 0.024506781643919416, "grpo_signal": 0.625, "out_tokens": 786, "invalid_steps": 0, "steps": 24, "pass@1": 0.0, "pass@k": 0.0, "k": 3 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_solve_rate": 0.0, "floor_policy": "null" }, "statistical.reward_reachability": { "pass": true, "reference": "beam-high", "pass@1": 1.0, "pass@k": 1.0, "k": 1 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 1.0, "ci95": [ 1.0, 1.0 ], "p": 0.0, "n": [ 8, 8 ], "between_beam_efforts": { "stronger": "beam-high", "weaker": "beam-high", "diff": 0.0, "ci95": [ 0.0, 0.0 ], "p": 1.0, "separates": false } }, "statistical.headroom": { "pass": false, "best_mean": 1.0, "max": 1.0, "room_fraction": 0.0 }, "trainee.reachability": { "pass": false, "pass@1": 0.0, "pass@k": 0.0, "k": 3 }, "trainee.grpo_signal": { "pass": true, "share": 0.625, "threshold": 0.2 }, "trainee.rule": { "prediction": "improves" }, "trainee.room_below_reference": { "pass": true, "reference_minus_trainee": 0.9710719650652249, "p": 0.0 }, "statistical.adversarial_floor": { "pass": true, "strategy": "constant", "label": "always \"5\"", "answer": "5", "score": 0.01, "learner_mean": 0.029, "added_after_training": true } }, "verdict": "worth training the small model on", "group": "worth", "reference": "beam-high", "trainee": "small", "judged_for": "small model", "tasks": 8, "provisional": false, "gameable": false, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": 0.0, "solved": 0.0, "grpo": null, "n": 8 }, { "policy": "small", "x": 0.0289, "mean": 0.028928034934775193, "solved": 0.0, "grpo": 0.625, "n": 24 }, { "policy": "beam-low", "x": 1.0, "mean": 1.0, "solved": 1.0, "grpo": 0.0, "n": 24 }, { "policy": "beam-high", "x": 1.0, "mean": 1.0, "solved": 1.0, "grpo": null, "n": 8 } ], "top_known": true } }, { "suite": "rg-countdown", "env": "reasoning_gym_env", "notes": "reasoning_gym family countdown", "episodes": 72, "errors": 0, "harness_errors": 0, "tiers": { "beam-high": { "episodes": 8, "mean": 1.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 41747, "invalid_steps": 0, "steps": 8, "pass@1": 1.0, "pass@k": 1.0, "k": 1 }, "beam-low": { "episodes": 24, "mean": 0.6833333333333332, "std": 0.44783429475148007, "within_task_std": 0.16793786053180504, "grpo_signal": 0.375, "out_tokens": 27951, "invalid_steps": 0, "steps": 24, "pass@1": 0.6666666666666666, "pass@k": 0.875, "k": 3 }, "beam-medium": { "episodes": 8, "mean": 0.7625, "std": 0.41136206679760834, "within_task_std": null, "grpo_signal": null, "out_tokens": 18989, "invalid_steps": 0, "steps": 8, "pass@1": 0.75, "pass@k": 0.75, "k": 1 }, "null": { "episodes": 8, "mean": 0.01, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 8, "pass@1": 0.0, "pass@k": 0.0, "k": 1 }, "small": { "episodes": 24, "mean": 0.03833333333333334, "std": 0.01818118685772619, "within_task_std": 0.007071067811865479, "grpo_signal": 0.375, "out_tokens": 295, "invalid_steps": 0, "steps": 24, "pass@1": 0.0, "pass@k": 0.0, "k": 3 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_solve_rate": 0.0, "floor_policy": "null" }, "statistical.reward_reachability": { "pass": true, "reference": "beam-high", "pass@1": 1.0, "pass@k": 1.0, "k": 1 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 0.99, "ci95": [ 0.99, 0.99 ], "p": 0.0, "n": [ 8, 8 ], "between_beam_efforts": { "stronger": "beam-high", "weaker": "beam-low", "diff": 0.31666666666666676, "ci95": [ 0.15833333333333321, 0.5145833333333334 ], "p": 0.0005, "separates": true } }, "statistical.headroom": { "pass": false, "best_mean": 1.0, "max": 1.0, "room_fraction": 0.0 }, "trainee.reachability": { "pass": false, "pass@1": 0.0, "pass@k": 0.0, "k": 3 }, "trainee.grpo_signal": { "pass": true, "share": 0.375, "threshold": 0.2 }, "trainee.rule": { "prediction": "improves" }, "trainee.room_below_reference": { "pass": true, "reference_minus_trainee": 0.9616666666666667, "p": 0.0 }, "statistical.adversarial_floor": { "pass": true, "strategy": "constant", "label": "always one fixed answer", "answer": "6*(98 - 54) - 34 - 66", "score": 0.05, "learner_mean": 0.038, "added_after_training": true } }, "verdict": "worth training the small model on", "group": "worth", "reference": "beam-high", "trainee": "small", "judged_for": "small model", "tasks": 8, "provisional": false, "gameable": false, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": 0.01, "solved": 0.0, "grpo": null, "n": 8 }, { "policy": "small", "x": 0.0286, "mean": 0.03833333333333334, "solved": 0.0, "grpo": 0.375, "n": 24 }, { "policy": "beam-low", "x": 0.6801, "mean": 0.6833333333333332, "solved": 0.6666666666666666, "grpo": 0.375, "n": 24 }, { "policy": "beam-medium", "x": 0.7601, "mean": 0.7625, "solved": 0.75, "grpo": null, "n": 8 }, { "policy": "beam-high", "x": 1.0, "mean": 1.0, "solved": 1.0, "grpo": null, "n": 8 } ], "top_known": true } }, { "suite": "rg-game_of_life", "env": "reasoning_gym_env", "notes": "reasoning_gym family game_of_life", "episodes": 80, "errors": 0, "harness_errors": 0, "tiers": { "beam-high": { "episodes": 8, "mean": 0.99625, "std": 0.009921567416492224, "within_task_std": null, "grpo_signal": null, "out_tokens": 59220, "invalid_steps": 0, "steps": 8, "pass@1": 0.875, "pass@k": 0.875, "k": 1 }, "beam-low": { "episodes": 24, "mean": 0.9775, "std": 0.03467107343785787, "within_task_std": 0.022835008406255256, "grpo_signal": 1.0, "out_tokens": 56359, "invalid_steps": 0, "steps": 24, "pass@1": 0.375, "pass@k": 0.875, "k": 3 }, "beam-max": { "episodes": 8, "mean": 0.9975, "std": 0.006614378277661483, "within_task_std": null, "grpo_signal": null, "out_tokens": 69050, "invalid_steps": 0, "steps": 8, "pass@1": 0.875, "pass@k": 0.875, "k": 1 }, "beam-medium": { "episodes": 8, "mean": 0.99875, "std": 0.0033071891388307415, "within_task_std": null, "grpo_signal": null, "out_tokens": 37490, "invalid_steps": 0, "steps": 8, "pass@1": 0.875, "pass@k": 0.875, "k": 1 }, "null": { "episodes": 8, "mean": 0.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 8, "pass@1": 0.0, "pass@k": 0.0, "k": 1 }, "small": { "episodes": 24, "mean": 0.0, "std": 0.0, "within_task_std": 0.0, "grpo_signal": 0.0, "out_tokens": 1536, "invalid_steps": 0, "steps": 24, "pass@1": 0.0, "pass@k": 0.0, "k": 3 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_solve_rate": 0.0, "floor_policy": "null" }, "statistical.reward_reachability": { "pass": true, "reference": "beam-medium", "pass@1": 0.875, "pass@k": 0.875, "k": 1 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 0.99875, "ci95": [ 0.9962500000000001, 1.0 ], "p": 0.0, "n": [ 8, 8 ], "between_beam_efforts": { "stronger": "beam-medium", "weaker": "beam-low", "diff": 0.02124999999999999, "ci95": [ 0.008739583333333266, 0.03708333333333336 ], "p": 0.0, "separates": true } }, "statistical.headroom": { "pass": false, "best_mean": 0.99875, "max": 1.0, "room_fraction": 0.0012499999999999734 }, "trainee.reachability": { "pass": false, "pass@1": 0.0, "pass@k": 0.0, "k": 3 }, "trainee.grpo_signal": { "pass": false, "share": 0.0, "threshold": 0.2 }, "trainee.rule": { "prediction": "too hard now" }, "trainee.room_below_reference": { "pass": true, "reference_minus_trainee": 0.99875, "p": 0.0 }, "statistical.adversarial_floor": { "pass": false, "strategy": "constant", "label": "always an empty board (every cell dead)", "answer": "[[0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0...", "score": 0.962, "learner_mean": 0.0, "added_after_training": true } }, "verdict": "too hard for the small model now: it earns no reward, while Beam solves 88%", "group": "hard", "reference": "beam-medium", "trainee": "small", "judged_for": "small model", "tasks": 8, "provisional": false, "gameable": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": 0.0, "solved": 0.0, "grpo": null, "n": 8 }, { "policy": "small", "x": 0.0, "mean": 0.0, "solved": 0.0, "grpo": 0.0, "n": 24 }, { "policy": "beam-low", "x": 0.9775, "mean": 0.9775, "solved": 0.375, "grpo": 1.0, "n": 24 }, { "policy": "beam-medium", "x": 0.9988, "mean": 0.99875, "solved": 0.875, "grpo": null, "n": 8 }, { "policy": "beam-high", "x": 0.9962, "mean": 0.99625, "solved": 0.875, "grpo": null, "n": 8 }, { "policy": "beam-max", "x": 0.9975, "mean": 0.9975, "solved": 0.875, "grpo": null, "n": 8 } ], "top_known": true } }, { "suite": "rg-leg_counting", "env": "reasoning_gym_env", "notes": "reasoning_gym family leg_counting", "episodes": 72, "errors": 0, "harness_errors": 0, "tiers": { "beam-high": { "episodes": 8, "mean": 1.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 9648, "invalid_steps": 0, "steps": 8, "pass@1": 1.0, "pass@k": 1.0, "k": 1 }, "beam-low": { "episodes": 24, "mean": 0.75, "std": 0.4330127018922193, "within_task_std": 0.23570226039551584, "grpo_signal": 0.5, "out_tokens": 6811, "invalid_steps": 0, "steps": 24, "pass@1": 0.75, "pass@k": 1.0, "k": 3 }, "beam-medium": { "episodes": 8, "mean": 0.75, "std": 0.4330127018922193, "within_task_std": null, "grpo_signal": null, "out_tokens": 4137, "invalid_steps": 0, "steps": 8, "pass@1": 0.75, "pass@k": 0.75, "k": 1 }, "null": { "episodes": 8, "mean": 0.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 8, "pass@1": 0.0, "pass@k": 0.0, "k": 1 }, "small": { "episodes": 24, "mean": 0.0, "std": 0.0, "within_task_std": 0.0, "grpo_signal": 0.0, "out_tokens": 1292, "invalid_steps": 0, "steps": 24, "pass@1": 0.0, "pass@k": 0.0, "k": 3 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_solve_rate": 0.0, "floor_policy": "null" }, "statistical.reward_reachability": { "pass": true, "reference": "beam-high", "pass@1": 1.0, "pass@k": 1.0, "k": 1 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 1.0, "ci95": [ 1.0, 1.0 ], "p": 0.0, "n": [ 8, 8 ], "between_beam_efforts": { "stronger": "beam-high", "weaker": "beam-low", "diff": 0.25, "ci95": [ 0.08333333333333337, 0.41666666666666663 ], "p": 0.00075, "separates": true } }, "statistical.headroom": { "pass": false, "best_mean": 1.0, "max": 1.0, "room_fraction": 0.0 }, "trainee.reachability": { "pass": false, "pass@1": 0.0, "pass@k": 0.0, "k": 3 }, "trainee.grpo_signal": { "pass": false, "share": 0.0, "threshold": 0.2 }, "trainee.rule": { "prediction": "too hard now" }, "trainee.room_below_reference": { "pass": true, "reference_minus_trainee": 1.0, "p": 0.0 }, "statistical.adversarial_floor": { "pass": true, "strategy": "constant", "label": "always \"184\"", "answer": "184", "score": 0.025, "learner_mean": 0.0, "added_after_training": true } }, "verdict": "too hard for the small model now: it earns no reward, while Beam solves 100%", "group": "hard", "reference": "beam-high", "trainee": "small", "judged_for": "small model", "tasks": 8, "provisional": false, "gameable": false, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": 0.0, "solved": 0.0, "grpo": null, "n": 8 }, { "policy": "small", "x": 0.0, "mean": 0.0, "solved": 0.0, "grpo": 0.0, "n": 24 }, { "policy": "beam-low", "x": 0.75, "mean": 0.75, "solved": 0.75, "grpo": 0.5, "n": 24 }, { "policy": "beam-medium", "x": 0.75, "mean": 0.75, "solved": 0.75, "grpo": null, "n": 8 }, { "policy": "beam-high", "x": 1.0, "mean": 1.0, "solved": 1.0, "grpo": null, "n": 8 } ], "top_known": true } }, { "suite": "rg-letter_counting", "env": "reasoning_gym_env", "notes": "reasoning_gym family letter_counting", "episodes": 64, "errors": 0, "harness_errors": 0, "tiers": { "beam-high": { "episodes": 8, "mean": 0.75, "std": 0.4330127018922193, "within_task_std": null, "grpo_signal": null, "out_tokens": 2997, "invalid_steps": 0, "steps": 8, "pass@1": 0.75, "pass@k": 0.75, "k": 1 }, "beam-low": { "episodes": 24, "mean": 0.6666666666666666, "std": 0.4714045207910317, "within_task_std": 0.35355339059327373, "grpo_signal": 0.75, "out_tokens": 4284, "invalid_steps": 0, "steps": 24, "pass@1": 0.6666666666666666, "pass@k": 1.0, "k": 3 }, "null": { "episodes": 8, "mean": 0.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 8, "pass@1": 0.0, "pass@k": 0.0, "k": 1 }, "small": { "episodes": 24, "mean": 0.04512210708291711, "std": 0.1993864118894454, "within_task_std": 0.06381229579951248, "grpo_signal": 0.5, "out_tokens": 425, "invalid_steps": 0, "steps": 24, "pass@1": 0.041666666666666664, "pass@k": 0.125, "k": 3 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_solve_rate": 0.0, "floor_policy": "null" }, "statistical.reward_reachability": { "pass": true, "reference": "beam-high", "pass@1": 0.75, "pass@k": 0.75, "k": 1 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 0.75, "ci95": [ 0.375, 1.0 ], "p": 0.0, "n": [ 8, 8 ], "between_beam_efforts": { "stronger": "beam-high", "weaker": "beam-low", "diff": 0.08333333333333337, "ci95": [ -0.29166666666666663, 0.41666666666666663 ], "p": 0.3505, "separates": false } }, "statistical.headroom": { "pass": true, "best_mean": 0.75, "max": 1.0, "room_fraction": 0.25 }, "trainee.reachability": { "pass": true, "pass@1": 0.041666666666666664, "pass@k": 0.125, "k": 3 }, "trainee.grpo_signal": { "pass": true, "share": 0.5, "threshold": 0.2 }, "trainee.rule": { "prediction": "improves" }, "trainee.room_below_reference": { "pass": true, "reference_minus_trainee": 0.7048778929170829, "p": 0.00025 }, "statistical.adversarial_floor": { "pass": false, "strategy": "constant", "label": "always \"1\"", "answer": "1", "score": 0.35, "learner_mean": 0.045, "added_after_training": true } }, "verdict": "worth training the small model on", "group": "worth", "reference": "beam-high", "trainee": "small", "judged_for": "small model", "tasks": 8, "provisional": false, "gameable": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": 0.0, "solved": 0.0, "grpo": null, "n": 8 }, { "policy": "small", "x": 0.0451, "mean": 0.04512210708291711, "solved": 0.041666666666666664, "grpo": 0.5, "n": 24 }, { "policy": "beam-low", "x": 0.6667, "mean": 0.6666666666666666, "solved": 0.6666666666666666, "grpo": 0.75, "n": 24 }, { "policy": "beam-high", "x": 0.75, "mean": 0.75, "solved": 0.75, "grpo": null, "n": 8 } ], "top_known": true } }, { "suite": "rg-maze", "env": "reasoning_gym_env", "notes": "reasoning_gym family maze", "episodes": 72, "errors": 0, "harness_errors": 0, "tiers": { "beam-high": { "episodes": 8, "mean": 1.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 31333, "invalid_steps": 0, "steps": 8, "pass@1": 1.0, "pass@k": 1.0, "k": 1 }, "beam-low": { "episodes": 24, "mean": 0.75, "std": 0.4330127018922193, "within_task_std": 0.11785113019775792, "grpo_signal": 0.25, "out_tokens": 29548, "invalid_steps": 0, "steps": 24, "pass@1": 0.75, "pass@k": 0.875, "k": 3 }, "beam-medium": { "episodes": 8, "mean": 0.875, "std": 0.33071891388307384, "within_task_std": null, "grpo_signal": null, "out_tokens": 18204, "invalid_steps": 0, "steps": 8, "pass@1": 0.875, "pass@k": 0.875, "k": 1 }, "null": { "episodes": 8, "mean": 0.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 8, "pass@1": 0.0, "pass@k": 0.0, "k": 1 }, "small": { "episodes": 24, "mean": 0.0008869368531945219, "std": 0.003216665538319247, "within_task_std": 0.0012543181267562077, "grpo_signal": 0.25, "out_tokens": 371, "invalid_steps": 0, "steps": 24, "pass@1": 0.0, "pass@k": 0.0, "k": 3 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_solve_rate": 0.0, "floor_policy": "null" }, "statistical.reward_reachability": { "pass": true, "reference": "beam-high", "pass@1": 1.0, "pass@k": 1.0, "k": 1 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 1.0, "ci95": [ 1.0, 1.0 ], "p": 0.0, "n": [ 8, 8 ], "between_beam_efforts": { "stronger": "beam-high", "weaker": "beam-low", "diff": 0.25, "ci95": [ 0.08333333333333337, 0.41666666666666663 ], "p": 0.00175, "separates": true } }, "statistical.headroom": { "pass": false, "best_mean": 1.0, "max": 1.0, "room_fraction": 0.0 }, "trainee.reachability": { "pass": false, "pass@1": 0.0, "pass@k": 0.0, "k": 3 }, "trainee.grpo_signal": { "pass": true, "share": 0.25, "threshold": 0.2 }, "trainee.rule": { "prediction": "improves" }, "trainee.room_below_reference": { "pass": true, "reference_minus_trainee": 0.9991130631468055, "p": 0.0 }, "statistical.adversarial_floor": { "pass": false, "strategy": "constant", "label": "always \"5\"", "answer": "5", "score": 0.315, "learner_mean": 0.001, "added_after_training": true } }, "verdict": "worth training the small model on", "group": "worth", "reference": "beam-high", "trainee": "small", "judged_for": "small model", "tasks": 8, "provisional": false, "gameable": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": 0.0, "solved": 0.0, "grpo": null, "n": 8 }, { "policy": "small", "x": 0.0009, "mean": 0.0008869368531945219, "solved": 0.0, "grpo": 0.25, "n": 24 }, { "policy": "beam-low", "x": 0.75, "mean": 0.75, "solved": 0.75, "grpo": 0.25, "n": 24 }, { "policy": "beam-medium", "x": 0.875, "mean": 0.875, "solved": 0.875, "grpo": null, "n": 8 }, { "policy": "beam-high", "x": 1.0, "mean": 1.0, "solved": 1.0, "grpo": null, "n": 8 } ], "top_known": true } }, { "suite": "rg-propositional_logic", "env": "reasoning_gym_env", "notes": "reasoning_gym family propositional_logic", "episodes": 72, "errors": 0, "harness_errors": 0, "tiers": { "beam-high": { "episodes": 8, "mean": 0.8125, "std": 0.3247595264191645, "within_task_std": null, "grpo_signal": null, "out_tokens": 18651, "invalid_steps": 0, "steps": 8, "pass@1": 0.75, "pass@k": 0.75, "k": 1 }, "beam-low": { "episodes": 24, "mean": 0.6875, "std": 0.369754986443726, "within_task_std": 0.13258252147247768, "grpo_signal": 0.375, "out_tokens": 8470, "invalid_steps": 0, "steps": 24, "pass@1": 0.5833333333333334, "pass@k": 0.75, "k": 3 }, "beam-medium": { "episodes": 8, "mean": 0.625, "std": 0.375, "within_task_std": null, "grpo_signal": null, "out_tokens": 5669, "invalid_steps": 0, "steps": 8, "pass@1": 0.5, "pass@k": 0.5, "k": 1 }, "null": { "episodes": 8, "mean": 0.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 8, "pass@1": 0.0, "pass@k": 0.0, "k": 1 }, "small": { "episodes": 24, "mean": 0.010416666666666666, "std": 0.020305821551685345, "within_task_std": 0.011785113019775792, "grpo_signal": 0.5, "out_tokens": 512, "invalid_steps": 0, "steps": 24, "pass@1": 0.0, "pass@k": 0.0, "k": 3 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_solve_rate": 0.0, "floor_policy": "null" }, "statistical.reward_reachability": { "pass": true, "reference": "beam-high", "pass@1": 0.75, "pass@k": 0.75, "k": 1 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 0.8125, "ci95": [ 0.53125, 1.0 ], "p": 0.0, "n": [ 8, 8 ], "between_beam_efforts": { "stronger": "beam-high", "weaker": "beam-medium", "diff": 0.1875, "ci95": [ -0.1875, 0.46875 ], "p": 0.204, "separates": false } }, "statistical.headroom": { "pass": true, "best_mean": 0.8125, "max": 1.0, "room_fraction": 0.1875 }, "trainee.reachability": { "pass": false, "pass@1": 0.0, "pass@k": 0.0, "k": 3 }, "trainee.grpo_signal": { "pass": true, "share": 0.5, "threshold": 0.2 }, "trainee.rule": { "prediction": "improves" }, "trainee.room_below_reference": { "pass": true, "reference_minus_trainee": 0.8020833333333334, "p": 0.0 }, "statistical.adversarial_floor": { "pass": false, "strategy": "replay input", "label": "the premises joined with 'and'", "answer": "the premises joined with 'and'", "score": 0.993, "learner_mean": 0.01, "added_after_training": true } }, "verdict": "worth training the small model on", "group": "worth", "reference": "beam-high", "trainee": "small", "judged_for": "small model", "tasks": 8, "provisional": false, "gameable": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": 0.0, "solved": 0.0, "grpo": null, "n": 8 }, { "policy": "small", "x": 0.0104, "mean": 0.010416666666666666, "solved": 0.0, "grpo": 0.5, "n": 24 }, { "policy": "beam-low", "x": 0.6875, "mean": 0.6875, "solved": 0.5833333333333334, "grpo": 0.375, "n": 24 }, { "policy": "beam-medium", "x": 0.625, "mean": 0.625, "solved": 0.5, "grpo": null, "n": 8 }, { "policy": "beam-high", "x": 0.8125, "mean": 0.8125, "solved": 0.75, "grpo": null, "n": 8 } ], "top_known": true } }, { "suite": "rg-sudoku", "env": "reasoning_gym_env", "notes": "reasoning_gym family sudoku", "episodes": 72, "errors": 1, "harness_errors": 0, "tiers": { "beam-high": { "episodes": 8, "mean": 1.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 116723, "invalid_steps": 0, "steps": 8, "pass@1": 1.0, "pass@k": 1.0, "k": 1 }, "beam-low": { "episodes": 24, "mean": 0.6660628476693008, "std": 0.2533264824118278, "within_task_std": 0.14614835638002605, "grpo_signal": 1.0, "out_tokens": 70465, "invalid_steps": 0, "steps": 24, "pass@1": 0.16666666666666666, "pass@k": 0.375, "k": 3 }, "beam-medium": { "episodes": 8, "mean": 0.8328189300411523, "std": 0.3335932120868177, "within_task_std": null, "grpo_signal": null, "out_tokens": 75140, "invalid_steps": 1, "steps": 8, "pass@1": 0.75, "pass@k": 0.75, "k": 1 }, "null": { "episodes": 8, "mean": 0.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 8, "pass@1": 0.0, "pass@k": 0.0, "k": 1 }, "small": { "episodes": 23, "mean": 0.17246376811594202, "std": 0.056071954119988054, "within_task_std": 0.01768579918330598, "grpo_signal": 1.0, "out_tokens": 1472, "invalid_steps": 0, "steps": 23, "pass@1": 0.0, "pass@k": 0.0, "k": 3 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_solve_rate": 0.0, "floor_policy": "null" }, "statistical.reward_reachability": { "pass": true, "reference": "beam-high", "pass@1": 1.0, "pass@k": 1.0, "k": 1 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 1.0, "ci95": [ 1.0, 1.0 ], "p": 0.0, "n": [ 8, 8 ], "between_beam_efforts": { "stronger": "beam-high", "weaker": "beam-low", "diff": 0.3339371523306992, "ci95": [ 0.23736650761190647, 0.44154141104294486 ], "p": 0.0, "separates": true } }, "statistical.headroom": { "pass": false, "best_mean": 1.0, "max": 1.0, "room_fraction": 0.0 }, "trainee.reachability": { "pass": false, "pass@1": 0.0, "pass@k": 0.0, "k": 3 }, "trainee.grpo_signal": { "pass": true, "share": 1.0, "threshold": 0.2 }, "trainee.rule": { "prediction": "improves" }, "trainee.room_below_reference": { "pass": true, "reference_minus_trainee": 0.827536231884058, "p": 0.0 }, "statistical.adversarial_floor": { "pass": true, "strategy": "constant", "label": "always one fixed answer", "answer": "8 6 2 1 9 3 5 4 7\n4 5 3 8 2 7 1 6 9\n1...", "score": 0.123, "learner_mean": 0.172, "added_after_training": true } }, "verdict": "worth training the small model on", "group": "worth", "reference": "beam-high", "trainee": "small", "judged_for": "small model", "tasks": 8, "provisional": false, "gameable": false, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": 0.0, "solved": 0.0, "grpo": null, "n": 8 }, { "policy": "small", "x": 0.1725, "mean": 0.17246376811594202, "solved": 0.0, "grpo": 1.0, "n": 23 }, { "policy": "beam-low", "x": 0.6661, "mean": 0.6660628476693008, "solved": 0.16666666666666666, "grpo": 1.0, "n": 24 }, { "policy": "beam-medium", "x": 0.8328, "mean": 0.8328189300411523, "solved": 0.75, "grpo": null, "n": 8 }, { "policy": "beam-high", "x": 1.0, "mean": 1.0, "solved": 1.0, "grpo": null, "n": 8 } ], "top_known": true } }, { "suite": "rg-word_sorting", "env": "reasoning_gym_env", "notes": "reasoning_gym family word_sorting", "episodes": 64, "errors": 0, "harness_errors": 0, "tiers": { "beam-high": { "episodes": 8, "mean": 0.9222222222222223, "std": 0.14185717038670784, "within_task_std": null, "grpo_signal": null, "out_tokens": 3119, "invalid_steps": 0, "steps": 8, "pass@1": 0.75, "pass@k": 0.75, "k": 1 }, "beam-low": { "episodes": 24, "mean": 0.9546296296296296, "std": 0.18036076862419545, "within_task_std": 0.06416339310766819, "grpo_signal": 0.25, "out_tokens": 5708, "invalid_steps": 0, "steps": 24, "pass@1": 0.9166666666666666, "pass@k": 1.0, "k": 3 }, "null": { "episodes": 8, "mean": 0.0, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 8, "pass@1": 0.0, "pass@k": 0.0, "k": 1 }, "small": { "episodes": 24, "mean": 0.20011574074074076, "std": 0.20387904779753585, "within_task_std": 0.0918835877736538, "grpo_signal": 0.875, "out_tokens": 451, "invalid_steps": 0, "steps": 24, "pass@1": 0.041666666666666664, "pass@k": 0.125, "k": 3 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_solve_rate": 0.0, "floor_policy": "null" }, "statistical.reward_reachability": { "pass": true, "reference": "beam-low", "pass@1": 0.9166666666666666, "pass@k": 1.0, "k": 3 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 0.9546296296296296, "ci95": [ 0.8722222222222221, 1.0 ], "p": 0.0, "n": [ 24, 8 ], "between_beam_efforts": { "stronger": "beam-low", "weaker": "beam-high", "diff": 0.03240740740740733, "ci95": [ -0.09074074074074068, 0.1611111111111112 ], "p": 0.3065, "separates": false } }, "statistical.headroom": { "pass": false, "best_mean": 0.9546296296296296, "max": 1.0, "room_fraction": 0.045370370370370394 }, "statistical.variance_bound": { "pass": true, "within_task_std": 0.06416339310766819, "signal": 0.9546296296296296 }, "trainee.reachability": { "pass": true, "pass@1": 0.041666666666666664, "pass@k": 0.125, "k": 3 }, "trainee.grpo_signal": { "pass": true, "share": 0.875, "threshold": 0.2 }, "trainee.rule": { "prediction": "improves" }, "trainee.room_below_reference": { "pass": true, "reference_minus_trainee": 0.7545138888888888, "p": 0.0 }, "statistical.adversarial_floor": { "pass": false, "strategy": "replay input", "label": "the input words, unsorted", "answer": "the input words, unsorted", "score": 0.256, "learner_mean": 0.2, "added_after_training": true } }, "verdict": "worth training the small model on", "group": "worth", "reference": "beam-low", "trainee": "small", "judged_for": "small model", "tasks": 8, "provisional": false, "gameable": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": 0.0, "solved": 0.0, "grpo": null, "n": 8 }, { "policy": "small", "x": 0.2001, "mean": 0.20011574074074076, "solved": 0.041666666666666664, "grpo": 0.875, "n": 24 }, { "policy": "beam-low", "x": 0.9546, "mean": 0.9546296296296296, "solved": 0.9166666666666666, "grpo": 0.25, "n": 24 }, { "policy": "beam-high", "x": 0.9222, "mean": 0.9222222222222223, "solved": 0.75, "grpo": null, "n": 8 } ], "top_known": true } }, { "suite": "snake_env", "env": "snake_env", "notes": "snake; capped at 20 moves", "episodes": 8, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": -0.15100000000000033, "std": 0.6924524051032912, "within_task_std": 0.6495158657126944, "grpo_signal": 1.0, "out_tokens": 39206, "invalid_steps": 0, "steps": 96 }, "null": { "episodes": 2, "mean": -0.9895, "std": 0.003500000000000003, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 23 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_mean": -0.9895, "top": 1.019999999999998, "gap_fraction": 0.9957879088206146 }, "statistical.reward_reachability": { "pass": true, "reference": "beam-low", "gain_over_floor": 0.8384999999999997, "ci95": [ 0.33233333333333326, 1.3433333333333324 ], "p": 0.002 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 0.8384999999999997, "ci95": [ 0.3311624999999997, 1.3449999999999989 ], "p": 0.00175, "n": [ 6, 2 ] }, "statistical.variance_bound": { "pass": true, "within_task_std": 0.6495158657126944, "signal": 0.8384999999999997 }, "statistical.grpo_signal": { "pass": true, "reference": "beam-low", "share": 1.0, "threshold": 0.2 } }, "verdict": "learnable for Beam; not yet tested with a small model", "group": "worth", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 2, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0042, "mean": -0.9895, "solved": null, "grpo": null, "n": 2 }, { "policy": "beam-low", "x": 0.4197, "mean": -0.15100000000000033, "solved": null, "grpo": 1.0, "n": 6 } ], "top_known": false } }, { "suite": "sophistry_bench_sprint_env", "env": "sophistry_bench_sprint_env", "notes": "single argued answer with verbatim citations", "episodes": 16, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 12, "mean": 0.9015624999999999, "std": 0.1377631241857438, "within_task_std": 0.08101206371139363, "grpo_signal": 0.75, "out_tokens": 7590, "invalid_steps": 0, "steps": 12 }, "null": { "episodes": 4, "mean": 0.5, "std": 0.0, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 4 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_mean": 0.5, "top": 1.0, "gap_fraction": 1.0 }, "statistical.reward_reachability": { "pass": true, "reference": "beam-low", "gain_over_floor": 0.40156249999999993, "ci95": [ 0.3151041666666665, 0.46875 ], "p": 0.0 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 0.40156249999999993, "ci95": [ 0.3135416666666666, 0.4671875 ], "p": 0.0, "n": [ 12, 4 ] }, "statistical.variance_bound": { "pass": true, "within_task_std": 0.08101206371139363, "signal": 0.40156249999999993 }, "statistical.grpo_signal": { "pass": true, "reference": "beam-low", "share": 0.75, "threshold": 0.2 } }, "verdict": "learnable for Beam; not yet tested with a small model", "group": "worth", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 4, "provisional": false, "rail": { "rungs": [ { "policy": "null", "x": 0.0, "mean": 0.5, "solved": null, "grpo": null, "n": 4 }, { "policy": "beam-low", "x": 0.8031, "mean": 0.9015624999999999, "solved": null, "grpo": 0.75, "n": 12 } ], "top_known": false } }, { "suite": "wildfire_env", "env": "wildfire_env", "notes": "128-step episode capped at 15, scored per step", "episodes": 8, "errors": 0, "harness_errors": 0, "tiers": { "beam-low": { "episodes": 6, "mean": -0.35797048611111104, "std": 0.5453663550613294, "within_task_std": 0.25352614683863084, "grpo_signal": 1.0, "out_tokens": 51941, "invalid_steps": 0, "steps": 69 }, "null": { "episodes": 2, "mean": -1.1283333333333334, "std": 0.07499999999999996, "within_task_std": null, "grpo_signal": null, "out_tokens": 0, "invalid_steps": 0, "steps": 30 } }, "checks": { "semantic.floor_gap": { "pass": true, "floor_mean": -1.1283333333333334, "top": 0.48908854166666665, "gap_fraction": 0.9556848081983105 }, "statistical.reward_reachability": { "pass": true, "reference": "beam-low", "gain_over_floor": 0.7703628472222224, "ci95": [ 0.33333333333333337, 1.2290312500000005 ], "p": 0.0 }, "statistical.difficulty_separation": { "pass": true, "reference_minus_floor": 0.7703628472222224, "ci95": [ 0.35955555555555574, 1.2428368055555556 ], "p": 0.0, "n": [ 6, 2 ] }, "statistical.variance_bound": { "pass": true, "within_task_std": 0.25352614683863084, "signal": 0.7703628472222224 }, "statistical.grpo_signal": { "pass": true, "reference": "beam-low", "share": 1.0, "threshold": 0.2 } }, "verdict": "learnable for Beam; not yet tested with a small model", "group": "worth", "reference": "beam-low", "trainee": null, "judged_for": "Beam", "tasks": 2, "provisional": true, "rail": { "rungs": [ { "policy": "null", "x": 0.0443, "mean": -1.1283333333333334, "solved": null, "grpo": null, "n": 2 }, { "policy": "beam-low", "x": 0.4995, "mean": -0.35797048611111104, "solved": null, "grpo": 1.0, "n": 6 } ], "top_known": false } } ], "connect4": { "env": "connect4_env", "as_shipped": { "null": { "won": 3, "games": 3 }, "random": { "won": 9, "games": 9 }, "beam-low": { "won": 9, "games": 9 } }, "with_beams_fix": { "null": { "won": 0, "games": 3 }, "random": { "won": 1, "games": 9 }, "beam-low": { "won": 8, "games": 9 } }, "diagnosis": "DIAGNOSIS:\nThe environment never makes an opponent move, so the agent effectively plays both sides, and `_check_win_or_draw` returns `+1.0` for any four-in-a-row regardless of which player made it. Every finished game therefore ends in a win and pays the top reward. The fix is to make the agent player 1, have a built-in opponent respond after each legal agent move, and score a win as `+1` only when player 1 wins and `-1` when the opponent wins.\n\nFIX:\nUpdated `server/connect4_environment.py` so each `step` places the agent's piece as player 1, then automatically plays an opponent move (immediate win/block/center preference) unless the game is already over. Reward polarity now depends on the winning player, and invalid moves still end the episode with `-1`." }, "dipg": { "default_markers": { "cleared_format_gate": 0, "answers": 12, "fully_correct": 0, "mean_score": -10.0 }, "square_brackets": { "cleared_format_gate": 7, "answers": 12, "fully_correct": 0, "mean_score": -4.17 }, "pilot": { "dipg_safety_env-brackets": { "cleared_format_gate": 6, "answers": 12, "fully_correct": 0, "mean_score": -5.0 }, "dipg_safety_env-plain": { "cleared_format_gate": 0, "answers": 12, "fully_correct": 0, "mean_score": -10.0 }, "dipg_safety_env": { "cleared_format_gate": 0, "answers": 12, "fully_correct": 0, "mean_score": -10.0 } } }, "effort": [ { "suite": "maze_env", "rungs": { "beam-low": { "solved": 1.0, "mean": 9.283, "episodes": 9 }, "beam-high": { "solved": 1.0, "mean": 9.35, "episodes": 3 } } }, { "suite": "os-tic_tac_toe", "rungs": { "beam-low": { "solved": 0.8888888888888888, "mean": 0.889, "episodes": 9 }, "beam-high": { "solved": 1.0, "mean": 1.0, "episodes": 3 } } }, { "suite": "rg-basic_arithmetic", "rungs": { "beam-low": { "solved": 0.875, "mean": 0.875, "episodes": 24 }, "beam-high": { "solved": 0.875, "mean": 0.875, "episodes": 8 } } }, { "suite": "rg-chain_sum", "rungs": { "beam-low": { "solved": 1.0, "mean": 1.0, "episodes": 24 }, "beam-high": { "solved": 1.0, "mean": 1.0, "episodes": 8 } } }, { "suite": "rg-countdown", "rungs": { "beam-low": { "solved": 0.6666666666666666, "mean": 0.683, "episodes": 24 }, "beam-medium": { "solved": 0.75, "mean": 0.762, "episodes": 8 }, "beam-high": { "solved": 1.0, "mean": 1.0, "episodes": 8 } } }, { "suite": "rg-game_of_life", "rungs": { "beam-low": { "solved": 0.375, "mean": 0.978, "episodes": 24 }, "beam-medium": { "solved": 0.875, "mean": 0.999, "episodes": 8 }, "beam-high": { "solved": 0.875, "mean": 0.996, "episodes": 8 }, "beam-max": { "solved": 0.875, "mean": 0.998, "episodes": 8 } } }, { "suite": "rg-leg_counting", "rungs": { "beam-low": { "solved": 0.75, "mean": 0.75, "episodes": 24 }, "beam-medium": { "solved": 0.75, "mean": 0.75, "episodes": 8 }, "beam-high": { "solved": 1.0, "mean": 1.0, "episodes": 8 } } }, { "suite": "rg-letter_counting", "rungs": { "beam-low": { "solved": 0.6666666666666666, "mean": 0.667, "episodes": 24 }, "beam-high": { "solved": 0.75, "mean": 0.75, "episodes": 8 } } }, { "suite": "rg-maze", "rungs": { "beam-low": { "solved": 0.75, "mean": 0.75, "episodes": 24 }, "beam-medium": { "solved": 0.875, "mean": 0.875, "episodes": 8 }, "beam-high": { "solved": 1.0, "mean": 1.0, "episodes": 8 } } }, { "suite": "rg-propositional_logic", "rungs": { "beam-low": { "solved": 0.5833333333333334, "mean": 0.688, "episodes": 24 }, "beam-medium": { "solved": 0.5, "mean": 0.625, "episodes": 8 }, "beam-high": { "solved": 0.75, "mean": 0.812, "episodes": 8 } } }, { "suite": "rg-sudoku", "rungs": { "beam-low": { "solved": 0.16666666666666666, "mean": 0.666, "episodes": 24 }, "beam-medium": { "solved": 0.75, "mean": 0.833, "episodes": 8 }, "beam-high": { "solved": 1.0, "mean": 1.0, "episodes": 8 } } }, { "suite": "rg-word_sorting", "rungs": { "beam-low": { "solved": 0.9166666666666666, "mean": 0.955, "episodes": 24 }, "beam-high": { "solved": 0.75, "mean": 0.922, "episodes": 8 } } } ], "effort_summary": { "suites": 12, "compared": 12, "higher_solved_more": 8, "same": 3, "higher_solved_less": 1 }, "census": { "spaces": 269, "groups": [ { "group": "running, but no standard OpenEnv API", "count": 79, "share": 0.294 }, { "group": "runs and passes the floor checks", "count": 63, "share": 0.234 }, { "group": "asleep, no answer within 10 minutes", "count": 36, "share": 0.134 }, { "group": "scores above zero for doing nothing", "count": 30, "share": 0.112 }, { "group": "broken (build or runtime error on the Hub)", "count": 25, "share": 0.093 }, { "group": "crashes on reset or step", "count": 17, "share": 0.063 }, { "group": "refuses a session", "count": 10, "share": 0.037 }, { "group": "paused by its owner", "count": 9, "share": 0.033 } ], "sum_of_shares": 1.0, "do_nothing_highest": [ { "space": "netra412005/flash-sale-env-v2", "do_nothing_score": 45275.3013708072 }, { "space": "tuhindev2029/cacheforge", "do_nothing_score": 12.7405 }, { "space": "stavust28/hospitalmanage_triage_env", "do_nothing_score": 7.000000000000001 }, { "space": "Jishnu-Vijayan-03/disaster-response", "do_nothing_score": 6.859200000000001 }, { "space": "Ev3Dev/Bio-EnvRL", "do_nothing_score": 5.977099354575067 } ], "do_nothing_at_least_1": 13, "second_pass_done": 156 }, "training": { "pre_registration": { "sha256": "d11a5928bce2f1b045b593e699370612779b6b45950cbdbee256ef6a23ccc776", "committed_utc": "2026-10-30 16:23:45", "first_checkpoint_utc": "2026-10-30 16:25" }, "held": 6, "families": 6, "rows": [ { "family": "basic_arithmetic", "prediction": "improves", "held": true, "reward_before": 0.052, "reward_after": 0.483, "p": 0.0, "accuracy_before": 0.01, "accuracy_after": 0.48, "lenient_accuracy_before": 0.255, "lenient_reward_before": 0.268, "baseline": { "strategy": "always answer 0", "score": 0.02 }, "learned": "skill and format", "learned_text": "Partly skill, partly format: read leniently, the untrained model was already right on 25.5%.", "beam_audit": "learned_format", "beam_explanation": "The before answers often contained the correct numeric result inside longer expressions or explanations, so the grader only gave partial credit through the substring branch. After training, the model emits the bare value, which the grader scores as an exact match. This indicates the gain is mainly from answer formatting rather than improved arithmetic." }, { "family": "chain_sum", "prediction": "improves", "held": true, "reward_before": 0.038, "reward_after": 0.3, "p": 0.0, "accuracy_before": 0.005, "accuracy_after": 0.23, "lenient_accuracy_before": 0.4, "lenient_reward_before": 0.4, "baseline": { "strategy": "always answer 0", "score": 0.02 }, "learned": "mostly format", "learned_text": "Mostly format: read leniently, the untrained model was already right on 40%.", "beam_audit": "learned_format", "beam_explanation": "The model improved mainly by emitting concise answers or calculation chains instead of verbose explanations, which raises string-similarity reward against the oracle answer. The same numeric answer can score very differently depending on surrounding text, as shown by the verbose versus concise versions of 757. Thus the gain is largely presentation-driven rather than evidence of better arithmetic." }, { "family": "letter_counting", "prediction": "improves", "held": true, "reward_before": 0.137, "reward_after": 0.35, "p": 0.0, "accuracy_before": 0.135, "accuracy_after": 0.35, "lenient_accuracy_before": null, "lenient_reward_before": null, "baseline": { "strategy": "always answer 1", "score": 0.35 }, "learned": "gamed", "learned_text": "It answers \"1\" to everything; that alone scores 0.350.", "beam_audit": "gamed_grader", "beam_explanation": "After training the model outputs the constant answer \"1\" on every shown task, so it only earns reward when the oracle count happens to be 1. This is not solving the counting task; it exploits the grader's exact-match reward for a frequent answer without depending on the question." }, { "family": "propositional_logic", "prediction": "improves", "held": true, "reward_before": 0.001, "reward_after": 0.993, "p": 0.0, "accuracy_before": 0.0, "accuracy_after": 0.99, "lenient_accuracy_before": null, "lenient_reward_before": null, "baseline": { "strategy": "join the premises with 'and'", "score": 0.993 }, "learned": "gamed", "learned_text": "It joins the premises with \"and\"; that alone scores 0.993.", "beam_audit": "gamed_grader", "beam_explanation": "The trained model does not appear to infer new conclusions; it emits conjunctions of premises and tautological variants, which are trivially entailed and receive full score. The pre-training answers often contained a plausible conclusion but were wrapped in prose, so much of the apparent gain comes from bypassing the parser and exploiting the grader's acceptance of premise conjunctions." }, { "family": "word_sorting", "prediction": "improves", "held": true, "reward_before": 0.221, "reward_after": 0.253, "p": 0.0076, "accuracy_before": 0.01, "accuracy_after": 0.02, "lenient_accuracy_before": null, "lenient_reward_before": null, "baseline": { "strategy": "repeat the input words unsorted", "score": 0.256 }, "learned": "no real gain", "learned_text": "Repeating the input words unsorted already scores 0.256.", "beam_audit": "gamed_grader", "beam_explanation": "The grader awards at least 0.2 whenever the answer contains the same words as the oracle, so repeating the input words can earn reward without actually sorting. Held-out scores remain near that 0.2 floor (0.221 to 0.253) and exact accuracy stays around 1-2%, while many shown answers are just input-order permutations scoring 0.2. This points to exploiting the grader loophole rather than learning sorting." }, { "family": "leg_counting", "prediction": "no change", "held": true, "reward_before": 0.0, "reward_after": 0.0, "p": 0.6534, "accuracy_before": 0.0, "accuracy_after": 0.0, "lenient_accuracy_before": null, "lenient_reward_before": null, "baseline": null, "learned": "no change", "learned_text": "No reward on any attempt, so nothing to learn from; this prediction could not fail.", "beam_audit": "no_change", "beam_explanation": "The held-out mean score stayed 0.000 before and after training, and the sampled answers remain incomplete or use incorrect leg counts. Training changed phrasing and often assigns six legs to every animal, but it did not produce answers that match the oracle or earn grader reward." } ] }, "forward_test": { "family": "maze", "written_at": "2026-10-31 22:31:12 UTC", "claims": [ { "id": "P1", "claim": "Reward rises.", "test": "Mean held-out reward after training is above before, paired bootstrap p <= 0.05 (the same test as the six-family run)." }, { "id": "P2", "claim": "The model collapses onto one constant answer.", "test": "After training, a single answer string makes up at least 50% of the 200 greedy held-out answers." }, { "id": "P3", "claim": "The constant is 5 or 6.", "test": "The most common answer after training is '5' or '6', the two most common correct answers on the training tasks." }, { "id": "P4", "claim": "No maze-solving skill is learned.", "test": "On held-out tasks whose correct answer differs from the model's most common answer after training, accuracy does not rise: paired bootstrap p > 0.05 for a gain." } ], "evidence": { "shortlist_verdict": "worth training the small model on", "small_model_mean_reward_on_ladder": 0.001, "small_model_grpo_signal_on_ladder": 0.25, "gaming_check": { "constant_answer": "5", "its_held_out_score": 0.315, "most_common_correct_answers_on_training_tasks": { "5": 114, "6": 112, "7": 66, "8": 52 }, "training_tasks": 400 } }, "sha256": "4cca90e80a446fcb9286c2e9007450c7fd31e2d692f32b24d256eca118477cc8", "result": { "collapsed_answer_start": "We are given a 5x5 grid:", "collapsed_answer_numbers": "1 to 20", "collapsed_answer_chars": 82, "collapsed_share": 0.98, "held_out_answers_inside_it": 200, "held_out_tasks": 200, "its_mean_score_if_given_everywhere": 0.013, "constant_5_score": 0.315, "reward_before": 0.004, "reward_after": 0.013, "exact_before": 0, "exact_after": 0, "bare_numbers_before": 20, "minutes": 26.4, "claims": [ { "id": "P1", "held": true, "claim": "Reward rises.", "test": "Mean held-out reward after training is above before, paired bootstrap p <= 0.05 (the same test as the six-family run).", "measured": "mean reward 0.004 \u2192 0.013, paired p < 0.001" }, { "id": "P2", "held": true, "claim": "The model collapses onto one constant answer.", "test": "After training, a single answer string makes up at least 50% of the 200 greedy held-out answers.", "measured": "196 of 200 answers are the same text" }, { "id": "P3", "held": false, "claim": "The constant is 5 or 6.", "test": "The most common answer after training is '5' or '6', the two most common correct answers on the training tasks.", "measured": "the constant is not a number: a grid of the numbers 1 to 20, cut off at 64 tokens" }, { "id": "P4", "held": true, "claim": "No maze-solving skill is learned.", "test": "On held-out tasks whose correct answer differs from the model's most common answer after training, accuracy does not rise: paired bootstrap p > 0.05 for a gain.", "measured": "exact answers 0 \u2192 0 of 200" } ] } }, "pelicans": [ { "svg_b64": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA2MDAgNDAwIiB3aWR0aD0iNjAwIiBoZWlnaHQ9IjQwMCI+CiAgPHJlY3Qgd2lkdGg9IjYwMCIgaGVpZ2h0PSI0MDAiIGZpbGw9IiNiZmVhZmYiLz4KICA8ZyBpZD0iYmljeWNsZSI+CiAgICA8Y2lyY2xlIGN4PSIxODAiIGN5PSIyODAiIHI9IjYwIiBmaWxsPSJub25lIiBzdHJva2U9IiMyMjIiIHN0cm9rZS13aWR0aD0iOCIvPgogICAgPGNpcmNsZSBjeD0iMTgwIiBjeT0iMjgwIiByPSI4IiBmaWxsPSIjMjIyIi8+CiAgICA8Y2lyY2xlIGN4PSI0MjAiIGN5PSIyODAiIHI9IjYwIiBmaWxsPSJub25lIiBzdHJva2U9IiMyMjIiIHN0cm9rZS13aWR0aD0iOCIvPgogICAgPGNpcmNsZSBjeD0iNDIwIiBjeT0iMjgwIiByPSI4IiBmaWxsPSIjMjIyIi8+CiAgICA8bGluZSB4MT0iMTgwIiB5MT0iMjgwIiB4Mj0iMzAwIiB5Mj0iMTkwIiBzdHJva2U9IiMzMzMiIHN0cm9rZS13aWR0aD0iNyIvPgogICAgPGxpbmUgeDE9IjMwMCIgeTE9IjE5MCIgeDI9IjQyMCIgeTI9IjI4MCIgc3Ryb2tlPSIjMzMzIiBzdHJva2Utd2lkdGg9IjciLz4KICAgIDxsaW5lIHgxPSIzMDAiIHkxPSIxOTAiIHgyPSIzMDAiIHkyPSIyNTAiIHN0cm9rZT0iIzMzMyIgc3Ryb2tlLXdpZHRoPSI3Ii8+CiAgICA8bGluZSB4MT0iMjgwIiB5MT0iMjUwIiB4Mj0iMzQwIiB5Mj0iMjUwIiBzdHJva2U9IiMzMzMiIHN0cm9rZS13aWR0aD0iNyIvPgogICAgPGxpbmUgeDE9IjQyMCIgeTE9IjI4MCIgeDI9IjQyMCIgeTI9IjIzMCIgc3Ryb2tlPSIjMzMzIiBzdHJva2Utd2lkdGg9IjciLz4KICAgIDxsaW5lIHgxPSI0MDAiIHkxPSIyMzAiIHgyPSI0NjAiIHkyPSIyMzAiIHN0cm9rZT0iIzMzMyIgc3Ryb2tlLXdpZHRoPSI3Ii8+CiAgPC9nPgogIDxnIGlkPSJwZWxpY2FuIj4KICAgIDxlbGxpcHNlIGN4PSIzMTAiIGN5PSIxNjUiIHJ4PSI2OCIgcnk9IjQ1IiBmaWxsPSIjZjJmMmYyIiBzdHJva2U9IiM1NTUiIHN0cm9rZS13aWR0aD0iMyIvPgogICAgPGVsbGlwc2UgY3g9IjI4MCIgY3k9IjE1NSIgcng9IjE4IiByeT0iMTYiIGZpbGw9IiNmMmYyZjIiIHN0cm9rZT0iIzU1NSIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgICA8Y2lyY2xlIGN4PSIyODgiIGN5PSIxNTAiIHI9IjQiIGZpbGw9IiMxMTEiLz4KICAgIDxwb2x5Z29uIHBvaW50cz0iMzAwLDE1OCA1MDAsMTUwIDUwMCwxNzIgMzE1LDE3OCIgZmlsbD0iI2Y3ZDc3NCIgc3Ryb2tlPSIjN2E2NDIwIiBzdHJva2Utd2lkdGg9IjIiLz4KICAgIDxlbGxpcHNlIGN4PSIzNjUiIGN5PSIxODIiIHJ4PSI0OCIgcnk9IjI0IiBmaWxsPSIjZThkMmE2IiBzdHJva2U9IiM3YTY0MjAiIHN0cm9rZS13aWR0aD0iMiIvPgogICAgPGVsbGlwc2UgY3g9IjMwNSIgY3k9IjE4NSIgcng9IjE0IiByeT0iOCIgZmlsbD0iI2U4ZDJhNiIgc3Ryb2tlPSIjN2E2NDIwIiBzdHJva2Utd2lkdGg9IjEuNSIvPgogIDwvZz4KPC9zdmc+", "reward": 1.0, "structure": 1.0, "judged": false, "policy": "beam-low" }, { "svg_b64": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyMjAiIGhlaWdodD0iMTUwIiB2aWV3Qm94PSIwIDAgMjIwIDE1MCI+CiAgPGcgc3Ryb2tlPSIjMTExIiBzdHJva2Utd2lkdGg9IjIiIGZpbGw9Im5vbmUiPgogICAgPGNpcmNsZSBjeD0iNjIiIGN5PSIxMTgiIHI9IjI1IiBmaWxsPSIjZThlOGU4Ii8+CiAgICA8Y2lyY2xlIGN4PSI2MiIgY3k9IjExOCIgcj0iNSIgZmlsbD0iIzMzMyIvPgogICAgPGNpcmNsZSBjeD0iMTU1IiBjeT0iMTE4IiByPSIyNSIgZmlsbD0iI2U4ZThlOCIvPgogICAgPGNpcmNsZSBjeD0iMTU1IiBjeT0iMTE4IiByPSI1IiBmaWxsPSIjMzMzIi8+CiAgICA8bGluZSB4MT0iNjIiIHkxPSIxMTgiIHgyPSI5MCIgeTI9Ijg1Ii8+CiAgICA8bGluZSB4MT0iOTAiIHkxPSI4NSIgeDI9IjEzMiIgeTI9Ijg1Ii8+CiAgICA8bGluZSB4MT0iMTMyIiB5MT0iODUiIHgyPSIxNTUiIHkyPSIxMTgiLz4KICAgIDxsaW5lIHgxPSI5MCIgeTE9Ijg1IiB4Mj0iNjIiIHkyPSI4NSIvPgogICAgPGxpbmUgeDE9IjYyIiB5MT0iODUiIHgyPSI2MiIgeTI9IjEwMiIvPgogICAgPGxpbmUgeDE9IjkwIiB5MT0iODUiIHgyPSI5MCIgeTI9IjcwIi8+CiAgICA8bGluZSB4MT0iMTMyIiB5MT0iODUiIHgyPSIxNDgiIHkyPSI3MiIvPgogICAgPGxpbmUgeDE9IjE0OCIgeTE9IjcyIiB4Mj0iMTYyIiB5Mj0iNzIiLz4KICA8L2c+CiAgPGcgZmlsbD0iIzcwOTBiOCIgc3Ryb2tlPSIjMjIyIiBzdHJva2Utd2lkdGg9IjIiPgogICAgPGVsbGlwc2UgY3g9Ijk1IiBjeT0iNjIiIHJ4PSIzNCIgcnk9IjE4Ii8+CiAgICA8Y2lyY2xlIGN4PSIxMjgiIGN5PSI0MiIgcj0iMTQiLz4KICA8L2c+CiAgPHBhdGggZD0iTTEzOSA0MCBDMTY1IDM4LCAxODggNDIsIDE5NiA0NSBMMTk4IDUwIEMxODYgNTYsIDE2NSA1OCwgMTQ0IDUyIFoiIGZpbGw9IiNmM2QxN2YiIHN0cm9rZT0iIzIyMiIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgPHBhdGggZD0iTTE0NCA1MiBDMTYwIDU4LCAxODAgNjAsIDE5MiA1OCBMMTkwIDY2IEMxNzQgNzAsIDE1NCA2NywgMTQwIDU4IFoiIGZpbGw9IiNlN2I4NmUiIHN0cm9rZT0iIzIyMiIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgPHBhdGggZD0iTTc2IDU0IEM2MCA0OCwgNTIgNjIsIDY2IDc0IEM4MiA3OCwgOTAgNjYsIDc2IDU0IFoiIGZpbGw9IiM4N2E4Y2UiIHN0cm9rZT0iIzIyMiIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgPGxpbmUgeDE9IjEyOCIgeTE9IjU2IiB4Mj0iMTA2IiB5Mj0iODIiIHN0cm9rZT0iIzIyMiIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgPGxpbmUgeDE9IjExMiIgeTE9IjYwIiB4Mj0iOTgiIHkyPSI4NSIgc3Ryb2tlPSIjMjIyIiBzdHJva2Utd2lkdGg9IjIiLz4KICA8ZWxsaXBzZSBjeD0iOTUiIGN5PSI3MiIgcng9IjgiIHJ5PSI1IiBmaWxsPSIjNGI2IiBzdHJva2U9IiMyMjIiIHN0cm9rZS13aWR0aD0iMiIvPgo8L3N2Zz4=", "reward": 1.0, "structure": 1.0, "judged": false, "policy": "beam-low" }, { "svg_b64": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MDAgMjYwIiB3aWR0aD0iNDAwIiBoZWlnaHQ9IjI2MCI+CiAgPGcgaWQ9ImJpY3ljbGUiPgogICAgPGNpcmNsZSBjeD0iMTEwIiBjeT0iMjAwIiByPSI0MCIgZmlsbD0ibm9uZSIgc3Ryb2tlPSIjMjIyIiBzdHJva2Utd2lkdGg9IjQiLz4KICAgIDxjaXJjbGUgY3g9IjI1MCIgY3k9IjIwMCIgcj0iNDAiIGZpbGw9Im5vbmUiIHN0cm9rZT0iIzIyMiIgc3Ryb2tlLXdpZHRoPSI0Ii8+CiAgICA8bGluZSB4MT0iMTEwIiB5MT0iMjAwIiB4Mj0iMTgwIiB5Mj0iMTMwIiBzdHJva2U9IiMyMjIiIHN0cm9rZS13aWR0aD0iNCIvPgogICAgPGxpbmUgeDE9IjE4MCIgeTE9IjEzMCIgeDI9IjI1MCIgeTI9IjIwMCIgc3Ryb2tlPSIjMjIyIiBzdHJva2Utd2lkdGg9IjQiLz4KICAgIDxsaW5lIHgxPSIxODAiIHkxPSIxMzAiIHgyPSIxODAiIHkyPSIxNjUiIHN0cm9rZT0iIzIyMiIgc3Ryb2tlLXdpZHRoPSI0Ii8+CiAgICA8bGluZSB4MT0iMTgwIiB5MT0iMTY1IiB4Mj0iMTUwIiB5Mj0iMTcwIiBzdHJva2U9IiMyMjIiIHN0cm9rZS13aWR0aD0iNCIvPgogICAgPGxpbmUgeDE9IjI1MCIgeTE9IjE0MCIgeDI9IjI5MCIgeTI9IjExMCIgc3Ryb2tlPSIjMjIyIiBzdHJva2Utd2lkdGg9IjQiLz4KICAgIDxsaW5lIHgxPSIyOTAiIHkxPSIxMTAiIHgyPSIyNzUiIHkyPSIxMDUiIHN0cm9rZT0iIzIyMiIgc3Ryb2tlLXdpZHRoPSI0Ii8+CiAgICA8cmVjdCB4PSIxNjUiIHk9IjEyMCIgd2lkdGg9IjM1IiBoZWlnaHQ9IjgiIHJ4PSI0IiBmaWxsPSIjNTUzMzExIi8+CiAgPC9nPgogIDxnIGlkPSJwZWxpY2FuIj4KICAgIDxlbGxpcHNlIGN4PSIxNzAiIGN5PSIxMjUiIHJ4PSI1NSIgcnk9IjM0IiBmaWxsPSIjZjBlNmQyIiBzdHJva2U9IiM2YjVhNDUiIHN0cm9rZS13aWR0aD0iMiIvPgogICAgPGNpcmNsZSBjeD0iMjI1IiBjeT0iOTgiIHI9IjI4IiBmaWxsPSIjZjBlNmQyIiBzdHJva2U9IiM2YjVhNDUiIHN0cm9rZS13aWR0aD0iMiIvPgogICAgPHBhdGggZD0iTTI0Miw5OCBMMzE4LDk0IEwyNDgsMTE2IFoiIGZpbGw9IiNmN2QzNTgiIHN0cm9rZT0iIzhhNmQyMiIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgICA8cGF0aCBkPSJNMjQ4LDExNCBRMjkwLDE0NSAyNDAsMTMyIFoiIGZpbGw9IiNmNWE5YjgiIHN0cm9rZT0iIzhhNmQyMiIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgICA8cGF0aCBkPSJNMTM1LDEwNSBRMTc1LDg1IDIxNSwxMDUgUTE3NSwxMjUgMTM1LDEwNSBaIiBmaWxsPSIjZDhjY2IwIiBzdHJva2U9IiM2YjVhNDUiIHN0cm9rZS13aWR0aD0iMiIvPgogICAgPGxpbmUgeDE9IjE2NSIgeTE9IjE1NSIgeDI9IjE3NSIgeTI9IjE4NSIgc3Ryb2tlPSIjNmI1YTQ1IiBzdHJva2Utd2lkdGg9IjMiLz4KICAgIDxsaW5lIHgxPSIxODUiIHkxPSIxNTUiIHgyPSIxOTUiIHkyPSIxODUiIHN0cm9rZT0iIzZiNWE0NSIgc3Ryb2tlLXdpZHRoPSIzIi8+CiAgICA8Y2lyY2xlIGN4PSIxNzUiIGN5PSIxOTAiIHI9IjUiIGZpbGw9IiM2YjVhNDUiLz4KICAgIDxjaXJjbGUgY3g9IjE5NSIgY3k9IjE5MCIgcj0iNSIgZmlsbD0iIzZiNWE0NSIvPgogIDwvZz4KPC9zdmc+", "reward": 1.0, "structure": 1.0, "judged": false, "policy": "beam-low" }, { "svg_b64": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MjAgMzIwIiB3aWR0aD0iNDIwIiBoZWlnaHQ9IjMyMCI+CiAgPGcgc3Ryb2tlPSIjMjIyIiBzdHJva2Utd2lkdGg9IjQiIGZpbGw9Im5vbmUiPgogICAgPGNpcmNsZSBjeD0iMTEwIiBjeT0iMjM1IiByPSI1OCIvPgogICAgPGNpcmNsZSBjeD0iMzEwIiBjeT0iMjM1IiByPSI1OCIvPgogICAgPGxpbmUgeDE9IjExMCIgeTE9IjIzNSIgeDI9IjIxMCIgeTI9IjE1MCIvPgogICAgPGxpbmUgeDE9IjMxMCIgeTE9IjIzNSIgeDI9IjIxMCIgeTI9IjE1MCIvPgogICAgPGxpbmUgeDE9IjIxMCIgeTE9IjE1MCIgeDI9IjIxMCIgeTI9IjExNSIvPgogICAgPGxpbmUgeDE9IjMxMCIgeTE9IjIzNSIgeDI9IjMxNSIgeTI9IjE4NSIvPgogICAgPGxpbmUgeDE9IjMxNSIgeTE9IjE4NSIgeDI9IjMwMCIgeTI9IjE1MCIvPgogICAgPGxpbmUgeDE9IjMwMCIgeTE9IjE1MCIgeDI9IjI4NSIgeTI9IjE0NSIvPgogIDwvZz4KICA8ZWxsaXBzZSBjeD0iMTk1IiBjeT0iMTI1IiByeD0iNjIiIHJ5PSI0MiIgZmlsbD0iI2YyZjJmMiIgc3Ryb2tlPSIjMjIyIiBzdHJva2Utd2lkdGg9IjMiLz4KICA8Y2lyY2xlIGN4PSIyMzUiIGN5PSI5OCIgcj0iMjgiIGZpbGw9IiNmMmYyZjIiIHN0cm9rZT0iIzIyMiIgc3Ryb2tlLXdpZHRoPSIzIi8+CiAgPHBhdGggZD0iTTI1NSA5MiBMMzQ1IDEwMiBMMjU1IDEyNCBaIiBmaWxsPSIjZjdjOTQ4IiBzdHJva2U9IiMyMjIiIHN0cm9rZS13aWR0aD0iMyIvPgogIDxlbGxpcHNlIGN4PSIyNTUiIGN5PSIxNDIiIHJ4PSIzNCIgcnk9IjI1IiBmaWxsPSIjZThiODlhIiBzdHJva2U9IiMyMjIiIHN0cm9rZS13aWR0aD0iMyIvPgogIDxjaXJjbGUgY3g9IjI0NiIgY3k9IjkyIiByPSI0IiBmaWxsPSIjMjIyIi8+CiAgPHBhdGggZD0iTTE0NSAxMjAgUTE3NSAxNjUgMjE1IDEyNSIgZmlsbD0iI2Q5ZDlkOSIgc3Ryb2tlPSIjMjIyIiBzdHJva2Utd2lkdGg9IjMiLz4KICA8cmVjdCB4PSIxOTUiIHk9IjE0OCIgd2lkdGg9IjgiIGhlaWdodD0iMzYiIGZpbGw9IiM2NjYiLz4KICA8ZWxsaXBzZSBjeD0iMzEwIiBjeT0iMjM1IiByeD0iOCIgcnk9IjgiIGZpbGw9IiMyMjIiLz4KICA8ZWxsaXBzZSBjeD0iMTEwIiBjeT0iMjM1IiByeD0iOCIgcnk9IjgiIGZpbGw9IiMyMjIiLz4KPC9zdmc+", "reward": 1.0, "structure": 1.0, "judged": false, "policy": "beam-low" }, { "svg_b64": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI0MDAiIGhlaWdodD0iMjIwIiB2aWV3Qm94PSIwIDAgNDAwIDIyMCI+CiAgPGNpcmNsZSBjeD0iMTAwIiBjeT0iMTcwIiByPSI0NSIgZmlsbD0ibm9uZSIgc3Ryb2tlPSJibGFjayIgc3Ryb2tlLXdpZHRoPSI1Ii8+CiAgPGNpcmNsZSBjeD0iMTAwIiBjeT0iMTcwIiByPSI2IiBmaWxsPSJibGFjayIvPgogIDxjaXJjbGUgY3g9IjI4MCIgY3k9IjE3MCIgcj0iNDUiIGZpbGw9Im5vbmUiIHN0cm9rZT0iYmxhY2siIHN0cm9rZS13aWR0aD0iNSIvPgogIDxjaXJjbGUgY3g9IjI4MCIgY3k9IjE3MCIgcj0iNiIgZmlsbD0iYmxhY2siLz4KICA8bGluZSB4MT0iMTAwIiB5MT0iMTcwIiB4Mj0iMTcwIiB5Mj0iMTIwIiBzdHJva2U9ImJsYWNrIiBzdHJva2Utd2lkdGg9IjUiLz4KICA8bGluZSB4MT0iMTcwIiB5MT0iMTIwIiB4Mj0iMjgwIiB5Mj0iMTcwIiBzdHJva2U9ImJsYWNrIiBzdHJva2Utd2lkdGg9IjUiLz4KICA8bGluZSB4MT0iMTcwIiB5MT0iMTIwIiB4Mj0iMTkwIiB5Mj0iMTAwIiBzdHJva2U9ImJsYWNrIiBzdHJva2Utd2lkdGg9IjUiLz4KICA8bGluZSB4MT0iMTkwIiB5MT0iMTAwIiB4Mj0iMjMwIiB5Mj0iODUiIHN0cm9rZT0iYmxhY2siIHN0cm9rZS13aWR0aD0iNSIvPgogIDxsaW5lIHgxPSIxOTAiIHkxPSIxMDAiIHgyPSIyMTAiIHkyPSI4MCIgc3Ryb2tlPSJibGFjayIgc3Ryb2tlLXdpZHRoPSI0Ii8+CiAgPHJlY3QgeD0iMTU1IiB5PSIxMTAiIHdpZHRoPSIzMCIgaGVpZ2h0PSI4IiBmaWxsPSJzYWRkbGVicm93biIvPgogIDxlbGxpcHNlIGN4PSIxNzAiIGN5PSI3NSIgcng9IjQ1IiByeT0iMjgiIGZpbGw9IndoaXRlIiBzdHJva2U9ImJsYWNrIiBzdHJva2Utd2lkdGg9IjIiLz4KICA8Y2lyY2xlIGN4PSIxMzUiIGN5PSI1MCIgcj0iMjQiIGZpbGw9IndoaXRlIiBzdHJva2U9ImJsYWNrIiBzdHJva2Utd2lkdGg9IjIiLz4KICA8cG9seWdvbiBwb2ludHM9IjExMSw0OCA3MCw1OCAxMTEsNjgiIGZpbGw9Im9yYW5nZSIgc3Ryb2tlPSJibGFjayIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgPGVsbGlwc2UgY3g9IjEwNSIgY3k9IjcyIiByeD0iMjAiIHJ5PSIxMiIgZmlsbD0ib3JhbmdlIiBzdHJva2U9ImJsYWNrIiBzdHJva2Utd2lkdGg9IjIiLz4KICA8ZWxsaXBzZSBjeD0iMTQwIiBjeT0iNzUiIHJ4PSI0IiByeT0iOCIgZmlsbD0iYmxhY2siLz4KICA8ZWxsaXBzZSBjeD0iMTU1IiBjeT0iOTUiIHJ4PSIxOCIgcnk9IjEwIiBmaWxsPSJibGFjayIvPgogIDxsaW5lIHgxPSIxOTUiIHkxPSI5NSIgeDI9IjIyNSIgeTI9IjE1MCIgc3Ryb2tlPSJibGFjayIgc3Ryb2tlLXdpZHRoPSI0Ii8+CiAgPGxpbmUgeDE9IjIyNSIgeTE9IjE1MCIgeDI9IjIzNSIgeTI9IjE2MCIgc3Ryb2tlPSJibGFjayIgc3Ryb2tlLXdpZHRoPSI0Ii8+CiAgPGxpbmUgeDE9IjE3NSIgeTE9Ijk4IiB4Mj0iMjAwIiB5Mj0iMTU1IiBzdHJva2U9ImJsYWNrIiBzdHJva2Utd2lkdGg9IjQiLz4KICA8bGluZSB4MT0iMjAwIiB5MT0iMTU1IiB4Mj0iMjE1IiB5Mj0iMTY1IiBzdHJva2U9ImJsYWNrIiBzdHJva2Utd2lkdGg9IjQiLz4KPC9zdmc+", "reward": 1.0, "structure": 1.0, "judged": false, "policy": "beam-low" }, { "svg_b64": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI1MTIiIGhlaWdodD0iMzg0IiB2aWV3Qm94PSIwIDAgNTEyIDM4NCI+CiAgPCEtLSBiaWN5Y2xlIC0tPgogIDxjaXJjbGUgY3g9IjEyMCIgY3k9IjI4MCIgcj0iNDgiIGZpbGw9Im5vbmUiIHN0cm9rZT0iIzMzMyIgc3Ryb2tlLXdpZHRoPSI2Ii8+CiAgPGNpcmNsZSBjeD0iMzYwIiBjeT0iMjgwIiByPSI0OCIgZmlsbD0ibm9uZSIgc3Ryb2tlPSIjMzMzIiBzdHJva2Utd2lkdGg9IjYiLz4KICA8bGluZSB4MT0iMTIwIiB5MT0iMjgwIiB4Mj0iMjQwIiB5Mj0iMjEwIiBzdHJva2U9IiMzMzMiIHN0cm9rZS13aWR0aD0iNiIvPgogIDxsaW5lIHgxPSIyNDAiIHkxPSIyMTAiIHgyPSIzNjAiIHkyPSIyODAiIHN0cm9rZT0iIzMzMyIgc3Ryb2tlLXdpZHRoPSI2Ii8+CiAgPGxpbmUgeDE9IjI0MCIgeTE9IjIxMCIgeDI9IjI0MCIgeTI9IjE1MCIgc3Ryb2tlPSIjMzMzIiBzdHJva2Utd2lkdGg9IjYiLz4KICA8bGluZSB4MT0iMjI1IiB5MT0iMTUwIiB4Mj0iMjc1IiB5Mj0iMTUwIiBzdHJva2U9IiMzMzMiIHN0cm9rZS13aWR0aD0iNiIvPgogIDxsaW5lIHgxPSIyNDAiIHkxPSIxNzAiIHgyPSIzMDAiIHkyPSIxNjAiIHN0cm9rZT0iIzMzMyIgc3Ryb2tlLXdpZHRoPSI1Ii8+CiAgPGNpcmNsZSBjeD0iMzA1IiBjeT0iMTYwIiByPSI4IiBmaWxsPSIjMzMzIi8+CiAgPCEtLSBwZWxpY2FuIGJvZHkgLS0+CiAgPGVsbGlwc2UgY3g9IjI1MCIgY3k9IjE0NSIgcng9IjU4IiByeT0iMzQiIGZpbGw9IiNmMmYyZjIiIHN0cm9rZT0iIzU1NSIgc3Ryb2tlLXdpZHRoPSIzIi8+CiAgPCEtLSBwZWxpY2FuIHRhaWwgLS0+CiAgPHBvbHlnb24gcG9pbnRzPSIxOTUsMTUwIDE2NSwxMzUgMTcwLDE2NSIgZmlsbD0iI2UwZTBlMCIgc3Ryb2tlPSIjNTU1IiBzdHJva2Utd2lkdGg9IjIiLz4KICA8IS0tIHBlbGljYW4gaGVhZCAtLT4KICA8Y2lyY2xlIGN4PSIzMDUiIGN5PSIxMDUiIHI9IjI2IiBmaWxsPSIjZjJmMmYyIiBzdHJva2U9IiM1NTUiIHN0cm9rZS13aWR0aD0iMyIvPgogIDwhLS0gbG9uZyBiZWFrIC0tPgogIDxwYXRoIGQ9Ik0zMjUsOTggQzM3NSw5MiA0MjUsODggNDY1LDkyIEw0NTAsMTEyIEM0MTAsMTA4IDM2NSwxMTIgMzI1LDExOCBaIiBmaWxsPSIjZjRjMDVhIiBzdHJva2U9IiNhNjdjMmQiIHN0cm9rZS13aWR0aD0iMiIvPgogIDwhLS0gdGhyb2F0IHBvdWNoIC0tPgogIDxwYXRoIGQ9Ik0zMjUsMTE4IEMzNDUsMTQwIDM3NSwxNDggMzk1LDEzOCBDMzgyLDEyNCAzNTUsMTE4IDMyNSwxMTggWiIgZmlsbD0iI2Q5OWE2YSIgc3Ryb2tlPSIjYTY3YzJkIiBzdHJva2Utd2lkdGg9IjIiLz4KICA8IS0tIGV5ZSAtLT4KICA8Y2lyY2xlIGN4PSIzMTAiIGN5PSIxMDAiIHI9IjQiIGZpbGw9IiMyMjIiLz4KICA8IS0tIG5lY2sgLS0+CiAgPHBhdGggZD0iTTI4NSwxMjUgQzI5NSwxMzggMjkwLDE1MCAyNzUsMTUwIiBmaWxsPSJub25lIiBzdHJva2U9IiM1NTUiIHN0cm9rZS13aWR0aD0iNCIvPgo8L3N2Zz4=", "reward": 1.0, "structure": 1.0, "judged": false, "policy": "beam-low" }, { "svg_b64": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI1MTIiIGhlaWdodD0iMzg0IiB2aWV3Qm94PSIwIDAgNTEyIDM4NCI+CiAgPCEtLSBiaWN5Y2xlIC0tPgogIDxjaXJjbGUgY3g9IjE1MCIgY3k9IjI4MCIgcj0iNjIiIGZpbGw9Im5vbmUiIHN0cm9rZT0iIzMzMyIgc3Ryb2tlLXdpZHRoPSI2Ii8+CiAgPGNpcmNsZSBjeD0iMzQwIiBjeT0iMjgwIiByPSI2MiIgZmlsbD0ibm9uZSIgc3Ryb2tlPSIjMzMzIiBzdHJva2Utd2lkdGg9IjYiLz4KICA8Y2lyY2xlIGN4PSIxNTAiIGN5PSIyODAiIHI9IjE0IiBmaWxsPSIjMzMzIi8+CiAgPGNpcmNsZSBjeD0iMzQwIiBjeT0iMjgwIiByPSIxNCIgZmlsbD0iIzMzMyIvPgogIDxsaW5lIHgxPSIxNTAiIHkxPSIyODAiIHgyPSIyMjgiIHkyPSIyMjIiIHN0cm9rZT0iIzMzMyIgc3Ryb2tlLXdpZHRoPSI3Ii8+CiAgPGxpbmUgeDE9IjIyOCIgeTE9IjIyMiIgeDI9IjI4MiIgeTI9IjIyMiIgc3Ryb2tlPSIjMzMzIiBzdHJva2Utd2lkdGg9IjciLz4KICA8bGluZSB4MT0iMjgyIiB5MT0iMjIyIiB4Mj0iMzQwIiB5Mj0iMjgwIiBzdHJva2U9IiMzMzMiIHN0cm9rZS13aWR0aD0iNyIvPgogIDxsaW5lIHgxPSIyMjgiIHkxPSIyMjIiIHgyPSIyMjgiIHkyPSIyNjgiIHN0cm9rZT0iIzMzMyIgc3Ryb2tlLXdpZHRoPSI3Ii8+CiAgPGxpbmUgeDE9IjIyOCIgeTE9IjI2OCIgeDI9IjM0MCIgeTI9IjI4MCIgc3Ryb2tlPSIjMzMzIiBzdHJva2Utd2lkdGg9IjciLz4KICA8bGluZSB4MT0iMjgyIiB5MT0iMjIyIiB4Mj0iMjgyIiB5Mj0iMTg0IiBzdHJva2U9IiMzMzMiIHN0cm9rZS13aWR0aD0iNiIvPgogIDxsaW5lIHgxPSIyODIiIHkxPSIxODQiIHgyPSIzMTgiIHkyPSIxNzYiIHN0cm9rZT0iIzMzMyIgc3Ryb2tlLXdpZHRoPSI2Ii8+CiAgPGNpcmNsZSBjeD0iMjI4IiBjeT0iMjY4IiByPSIxMiIgZmlsbD0iIzMzMyIvPgogIDxsaW5lIHgxPSIyMjgiIHkxPSIyMjIiIHgyPSIyNDgiIHkyPSIxODgiIHN0cm9rZT0iIzMzMyIgc3Ryb2tlLXdpZHRoPSI1Ii8+CiAgPHJlY3QgeD0iMjM2IiB5PSIxODAiIHdpZHRoPSIzNCIgaGVpZ2h0PSIxMCIgcng9IjUiIGZpbGw9IiM1NTUiLz4KCiAgPCEtLSBwZWxpY2FuIC0tPgogIDxlbGxpcHNlIGN4PSIyNTAiIGN5PSIxODYiIHJ4PSI0OCIgcnk9IjM0IiBmaWxsPSIjZjVmMGQ4IiBzdHJva2U9IiM5OTkiIHN0cm9rZS13aWR0aD0iMyIvPgogIDxlbGxpcHNlIGN4PSIzMDIiIGN5PSIxNDYiIHJ4PSIzMiIgcnk9IjI2IiBmaWxsPSIjZjVmMGQ4IiBzdHJva2U9IiM5OTkiIHN0cm9rZS13aWR0aD0iMyIvPgogIDxwb2x5Z29uIHBvaW50cz0iMjcwLDE0MiAzNjAsMTI4IDQwMiwxNDUgMzU4LDE1OCAyNzAsMTU2IiBmaWxsPSIjZmZjYzY2IiBzdHJva2U9IiNiODg2MGIiIHN0cm9rZS13aWR0aD0iMyIvPgogIDxwYXRoIGQ9Ik0yODggMTU4IFEzMDggMTk0IDM0OCAxOTAgUTM3NiAxODYgMzYwIDE1NCIgZmlsbD0iI2U4YjA0YSIgc3Ryb2tlPSIjYjg4NjBiIiBzdHJva2Utd2lkdGg9IjMiLz4KICA8Y2lyY2xlIGN4PSIzMjAiIGN5PSIxMzQiIHI9IjYiIGZpbGw9IiMxMTEiLz4KICA8cGF0aCBkPSJNMjA2IDE4NiBRMTgyIDIwNiAxNzAgMjM4IFEyMDYgMjM0IDIzMCAyMTAgWiIgZmlsbD0iI2Q4Y2ZiMCIgc3Ryb2tlPSIjOTk5IiBzdHJva2Utd2lkdGg9IjIiLz4KICA8cGF0aCBkPSJNMjM1IDIxNCBRMjU1IDIzOCAyODIgMjMyIFEzMDIgMjI4IDI5NiAyMDgiIGZpbGw9IiNkOGNmYjAiIHN0cm9rZT0iIzk5OSIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgPGxpbmUgeDE9IjI0OCIgeTE9IjIxNCIgeDI9IjI0NCIgeTI9IjI1OCIgc3Ryb2tlPSIjOTk5IiBzdHJva2Utd2lkdGg9IjUiLz4KICA8bGluZSB4MT0iMjY2IiB5MT0iMjE2IiB4Mj0iMjY0IiB5Mj0iMjY4IiBzdHJva2U9IiM5OTkiIHN0cm9rZS13aWR0aD0iNSIvPgogIDxsaW5lIHgxPSIyNDQiIHkxPSIyNTgiIHgyPSIyMzAiIHkyPSIyODAiIHN0cm9rZT0iIzk5OSIgc3Ryb2tlLXdpZHRoPSI1Ii8+CiAgPGxpbmUgeDE9IjI2NCIgeTE9IjI2OCIgeDI9IjI3OCIgeTI9IjI4NiIgc3Ryb2tlPSIjOTk5IiBzdHJva2Utd2lkdGg9IjUiLz4KICA8bGluZSB4MT0iMjc4IiB5MT0iMjg2IiB4Mj0iMzEyIiB5Mj0iMjgwIiBzdHJva2U9IiM5OTkiIHN0cm9rZS13aWR0aD0iNSIvPgo8L3N2Zz4=", "reward": 1.0, "structure": 1.0, "judged": false, "policy": "beam-low" }, { "svg_b64": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MDAgMzAwIiB3aWR0aD0iNDAwIiBoZWlnaHQ9IjMwMCI+CiAgPCEtLSBiaWN5Y2xlIC0tPgogIDxnIGlkPSJiaWtlIj4KICAgIDwhLS0gd2hlZWxzIC0tPgogICAgPGNpcmNsZSBjeD0iMTIwIiBjeT0iMjQwIiByPSI0NSIgZmlsbD0ibm9uZSIgc3Ryb2tlPSIjMzMzIiBzdHJva2Utd2lkdGg9IjYiLz4KICAgIDxjaXJjbGUgY3g9IjEyMCIgY3k9IjI0MCIgcj0iMzAiIGZpbGw9Im5vbmUiIHN0cm9rZT0iIzg4OCIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgICA8Y2lyY2xlIGN4PSIyODAiIGN5PSIyNDAiIHI9IjQ1IiBmaWxsPSJub25lIiBzdHJva2U9IiMzMzMiIHN0cm9rZS13aWR0aD0iNiIvPgogICAgPGNpcmNsZSBjeD0iMjgwIiBjeT0iMjQwIiByPSIzMCIgZmlsbD0ibm9uZSIgc3Ryb2tlPSIjODg4IiBzdHJva2Utd2lkdGg9IjIiLz4KICAgIDwhLS0gZnJhbWUgLS0+CiAgICA8bGluZSB4MT0iMTIwIiB5MT0iMjQwIiB4Mj0iMjAwIiB5Mj0iMTgwIiBzdHJva2U9IiNjMDM5MmIiIHN0cm9rZS13aWR0aD0iNSIvPgogICAgPGxpbmUgeDE9IjIwMCIgeTE9IjE4MCIgeDI9IjI4MCIgeTI9IjI0MCIgc3Ryb2tlPSIjYzAzOTJiIiBzdHJva2Utd2lkdGg9IjUiLz4KICAgIDxsaW5lIHgxPSIyMDAiIHkxPSIxODAiIHgyPSIyMDAiIHkyPSIxNTAiIHN0cm9rZT0iI2MwMzkyYiIgc3Ryb2tlLXdpZHRoPSI1Ii8+CiAgICA8bGluZSB4MT0iMTYwIiB5MT0iMjQwIiB4Mj0iMjAwIiB5Mj0iMTgwIiBzdHJva2U9IiNjMDM5MmIiIHN0cm9rZS13aWR0aD0iNSIvPgogICAgPCEtLSBoYW5kbGViYXJzIC0tPgogICAgPGxpbmUgeDE9IjIwMCIgeTE9IjE1MCIgeDI9IjIxNSIgeTI9IjEzNSIgc3Ryb2tlPSIjMzMzIiBzdHJva2Utd2lkdGg9IjQiLz4KICAgIDxsaW5lIHgxPSIyMTUiIHkxPSIxMzUiIHgyPSIyMzAiIHkyPSIxMzUiIHN0cm9rZT0iIzMzMyIgc3Ryb2tlLXdpZHRoPSI0Ii8+CiAgICA8IS0tIHNhZGRsZSAtLT4KICAgIDxyZWN0IHg9IjE5MCIgeT0iMTQ1IiB3aWR0aD0iMzAiIGhlaWdodD0iOCIgcng9IjMiIGZpbGw9IiM1NTUiLz4KICAgIDwhLS0gcGVkYWxzIC0tPgogICAgPGNpcmNsZSBjeD0iMjAwIiBjeT0iMjEwIiByPSI2IiBmaWxsPSIjMzMzIi8+CiAgPC9nPgogIDwhLS0gcGVsaWNhbiAtLT4KICA8ZyBpZD0icGVsaWNhbiI+CiAgICA8IS0tIGJvZHkgLS0+CiAgICA8ZWxsaXBzZSBjeD0iMjAwIiBjeT0iMTMwIiByeD0iMzUiIHJ5PSIyMiIgZmlsbD0iI2Y0ZjRmNCIgc3Ryb2tlPSIjOTk5IiBzdHJva2Utd2lkdGg9IjEuNSIvPgogICAgPCEtLSB3aW5nIC0tPgogICAgPGVsbGlwc2UgY3g9IjE4NSIgY3k9IjEzMiIgcng9IjE4IiByeT0iOCIgZmlsbD0iI2QwZDBkMCIgc3Ryb2tlPSIjOTk5IiBzdHJva2Utd2lkdGg9IjEiLz4KICAgIDwhLS0gbmVjayAtLT4KICAgIDxwYXRoIGQ9Ik0gMjI1IDEyMCBRIDI2MCA5MCAyNjUgNjAiIHN0cm9rZT0iI2Y0ZjRmNCIgc3Ryb2tlLXdpZHRoPSIxMiIgZmlsbD0ibm9uZSIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIi8+CiAgICA8IS0tIGhlYWQgLS0+CiAgICA8Y2lyY2xlIGN4PSIyNjgiIGN5PSI1OCIgcj0iMTQiIGZpbGw9IiNmNGY0ZjQiIHN0cm9rZT0iIzk5OSIgc3Ryb2tlLXdpZHRoPSIxLjUiLz4KICAgIDwhLS0gYmVhayBsb25nIC0tPgogICAgPHBhdGggZD0iTSAyNzggNTUgTCAzMzUgNDggTCAzMzUgNjIgTCAyNzggNjUgWiIgZmlsbD0iI2YwYjQyOSIgc3Ryb2tlPSIjYzQ4YTAwIiBzdHJva2Utd2lkdGg9IjEiLz4KICAgIDwhLS0gdGhyb2F0IHBvdWNoIHVuZGVyIGJlYWsgLS0+CiAgICA8cGF0aCBkPSJNIDI4NSA2NSBRIDMxNSA3OCAzMzAgNjIiIGZpbGw9IiNmMGI0MjkiIHN0cm9rZT0iI2M0OGEwMCIgc3Ryb2tlLXdpZHRoPSIxIi8+CiAgICA8IS0tIGV5ZSAtLT4KICAgIDxjaXJjbGUgY3g9IjI3NSIgY3k9IjUyIiByPSIyLjUiIGZpbGw9IiMyMjIiLz4KICAgIDwhLS0gbGVncyAtLT4KICAgIDxsaW5lIHgxPSIxODUiIHkxPSIxNDgiIHgyPSIxODAiIHkyPSIyMDUiIHN0cm9rZT0iI2YwYjQyOSIgc3Ryb2tlLXdpZHRoPSIzIi8+CiAgICA8bGluZSB4MT0iMjEwIiB5MT0iMTQ4IiB4Mj0iMjA1IiB5Mj0iMjA1IiBzdHJva2U9IiNmMGI0MjkiIHN0cm9rZS13aWR0aD0iMyIvPgogICAgPCEtLSBmZWV0IC0tPgogICAgPGVsbGlwc2UgY3g9IjE3NSIgY3k9IjIwOCIgcng9IjgiIHJ5PSIzIiBmaWxsPSIjZjBiNDI5Ii8+CiAgICA8ZWxsaXBzZSBjeD0iMjAwIiBjeT0iMjA4IiByeD0iOCIgcnk9IjMiIGZpbGw9IiNmMGI0MjkiLz4KICA8L2c+Cjwvc3ZnPg==", "reward": 1.0, "structure": 1.0, "judged": false, "policy": "beam-low" }, { "svg_b64": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI0MjAiIGhlaWdodD0iMzIwIiB2aWV3Qm94PSIwIDAgNDIwIDMyMCI+CiAgPHJlY3Qgd2lkdGg9IjQyMCIgaGVpZ2h0PSIzMjAiIGZpbGw9IiNiZmU3ZmYiLz4KICA8ZWxsaXBzZSBjeD0iMjEwIiBjeT0iMzAwIiByeD0iMTkwIiByeT0iMjIiIGZpbGw9IiM3ZmM5N2YiLz4KICA8IS0tIGJpY3ljbGUgd2hlZWxzIC0tPgogIDxjaXJjbGUgY3g9IjExNSIgY3k9IjIzNSIgcj0iNTIiIGZpbGw9Im5vbmUiIHN0cm9rZT0iIzIyMiIgc3Ryb2tlLXdpZHRoPSI4Ii8+CiAgPGNpcmNsZSBjeD0iMTE1IiBjeT0iMjM1IiByPSI4IiBmaWxsPSIjNTU1Ii8+CiAgPGNpcmNsZSBjeD0iMzA1IiBjeT0iMjM1IiByPSI1MiIgZmlsbD0ibm9uZSIgc3Ryb2tlPSIjMjIyIiBzdHJva2Utd2lkdGg9IjgiLz4KICA8Y2lyY2xlIGN4PSIzMDUiIGN5PSIyMzUiIHI9IjgiIGZpbGw9IiM1NTUiLz4KICA8IS0tIGJpY3ljbGUgZnJhbWUgLS0+CiAgPHBvbHlnb24gcG9pbnRzPSIxMTUsMjM1IDIwNSwxNjUgMzA1LDIzNSAyMTUsMjM1IiBmaWxsPSJub25lIiBzdHJva2U9IiMyYjVmOGYiIHN0cm9rZS13aWR0aD0iNyIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCIvPgogIDxsaW5lIHgxPSIyMDUiIHkxPSIxNjUiIHgyPSIyMzUiIHkyPSIyMDUiIHN0cm9rZT0iIzJiNWY4ZiIgc3Ryb2tlLXdpZHRoPSI3Ii8+CiAgPGxpbmUgeDE9IjIzNSIgeTE9IjIwNSIgeDI9IjMwNSIgeTI9IjIzNSIgc3Ryb2tlPSIjMmI1ZjhmIiBzdHJva2Utd2lkdGg9IjciLz4KICA8IS0tIHNlYXQgLS0+CiAgPGVsbGlwc2UgY3g9IjIxNSIgY3k9IjE1OCIgcng9IjI0IiByeT0iMTAiIGZpbGw9IiM2YjRmMmEiLz4KICA8IS0tIGhhbmRsZWJhcnMgLS0+CiAgPGxpbmUgeDE9IjMwNSIgeTE9IjIzNSIgeDI9IjMwNSIgeTI9IjE2MCIgc3Ryb2tlPSIjMmI1ZjhmIiBzdHJva2Utd2lkdGg9IjciLz4KICA8cG9seWxpbmUgcG9pbnRzPSIyODgsMTYwIDMyMiwxNjAgMzIyLDE0NSIgZmlsbD0ibm9uZSIgc3Ryb2tlPSIjMjIyIiBzdHJva2Utd2lkdGg9IjYiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIvPgogIDwhLS0gcGVsaWNhbiBib2R5IC0tPgogIDxlbGxpcHNlIGN4PSIyMjUiIGN5PSIxMjgiIHJ4PSI1OCIgcnk9IjM2IiBmaWxsPSIjZjdmMmU3IiBzdHJva2U9IiM5OTkiIHN0cm9rZS13aWR0aD0iMiIvPgogIDxlbGxpcHNlIGN4PSIxOTUiIGN5PSIxMjIiIHJ4PSIyNCIgcnk9IjE4IiBmaWxsPSIjZmZmZmZmIiBvcGFjaXR5PSIwLjU1Ii8+CiAgPCEtLSB3aW5nIC0tPgogIDxwYXRoIGQ9Ik0xOTUsMTMyIEMyMDUsMTUwIDI1MCwxNTUgMjc1LDEzNSBDMjU1LDEzOCAyMjUsMTQyIDE5NSwxMzIgWiIgZmlsbD0iI2RkZDZjNCIvPgogIDwhLS0gbmVjayAtLT4KICA8cGF0aCBkPSJNMjY1LDExOCBDMjkwLDg1IDMwNSw3MCAzMTUsNjIiIGZpbGw9Im5vbmUiIHN0cm9rZT0iI2Y3ZjJlNyIgc3Ryb2tlLXdpZHRoPSIxOCIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIi8+CiAgPCEtLSBoZWFkIC0tPgogIDxlbGxpcHNlIGN4PSIzMjUiIGN5PSI1MiIgcng9IjI2IiByeT0iMTgiIGZpbGw9IiNmN2YyZTciIHN0cm9rZT0iIzk5OSIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgPGNpcmNsZSBjeD0iMzMzIiBjeT0iNDciIHI9IjMuNSIgZmlsbD0iIzExMSIvPgogIDwhLS0gbG9uZyBiZWFrIC0tPgogIDxwb2x5Z29uIHBvaW50cz0iMzQ2LDQ4IDQwNSw1OCAzNDYsNjgiIGZpbGw9IiNmMGEzM2IiIHN0cm9rZT0iI2I2NzMxOCIgc3Ryb2tlLXdpZHRoPSIyIi8+CiAgPCEtLSB0aHJvYXQgcG91Y2ggdW5kZXIgYmVhayAtLT4KICA8ZWxsaXBzZSBjeD0iMzUwIiBjeT0iNzYiIHJ4PSIzNCIgcnk9IjIwIiBmaWxsPSIjZjdkNmEzIiBzdHJva2U9IiNiNjczMTgiIHN0cm9rZS13aWR0aD0iMiIvPgogIDxwYXRoIGQ9Ik0zMzQsNjYgQzM0OCw4NCAzNzAsODYgMzg2LDcwIiBmaWxsPSJub25lIiBzdHJva2U9IiNkOThiMmIiIHN0cm9rZS13aWR0aD0iMiIvPgogIDwhLS0gbGVncyBhbmQgcGVkYWxzIC0tPgogIDxsaW5lIHgxPSIyMDUiIHkxPSIxNTYiIHgyPSIxODUiIHkyPSIxODUiIHN0cm9rZT0iI2NmYTA1YSIgc3Ryb2tlLXdpZHRoPSI1IiBzdHJva2UtbGluZWNhcD0icm91bmQiLz4KICA8bGluZSB4MT0iMjQ1IiB5MT0iMTU2IiB4Mj0iMjY1IiB5Mj0iMTg1IiBzdHJva2U9IiNjZmEwNWEiIHN0cm9rZS13aWR0aD0iNSIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIi8+CiAgPGNpcmNsZSBjeD0iMjI4IiBjeT0iMjA1IiByPSIxMiIgZmlsbD0iIzQ0NCIvPgogIDxlbGxpcHNlIGN4PSIyMjgiIGN5PSIyMDUiIHJ4PSI1IiByeT0iNSIgZmlsbD0iIzk5OSIvPgogIDwhLS0gc21hbGwgdGFpbCAtLT4KICA8cGF0aCBkPSJNMTcwLDEyOCBDMTQ1LDEyMCAxNDAsMTQ1IDE2MiwxNTIiIGZpbGw9IiNkZGQ2YzQiLz4KPC9zdmc+", "reward": 1.0, "structure": 1.0, "judged": false, "policy": "beam-low" } ], "links": { "dataset": "https://huggingface.co/datasets/JagritGumber/shortlist", "code": "https://huggingface.co/datasets/JagritGumber/shortlist/tree/main/code", "patch": "https://huggingface.co/datasets/JagritGumber/shortlist/blob/main/fixes/connect4_env.patch", "training": "https://huggingface.co/datasets/JagritGumber/shortlist/blob/main/training/RESULTS.md", "rfc": "https://github.com/openenv/OpenEnv/blob/main/rfcs/008-environment-auto-validation.md" } }