| { |
| "all_three_models_nonincreasing": true, |
| "assessment": "verified", |
| "claim": 5, |
| "combination_count": 15, |
| "complete_four_module_combinations": [ |
| [ |
| "loss" |
| ], |
| [ |
| "networks" |
| ], |
| [ |
| "optim" |
| ], |
| [ |
| "train" |
| ], |
| [ |
| "loss", |
| "networks" |
| ], |
| [ |
| "loss", |
| "optim" |
| ], |
| [ |
| "loss", |
| "train" |
| ], |
| [ |
| "networks", |
| "optim" |
| ], |
| [ |
| "networks", |
| "train" |
| ], |
| [ |
| "optim", |
| "train" |
| ], |
| [ |
| "loss", |
| "networks", |
| "optim" |
| ], |
| [ |
| "loss", |
| "networks", |
| "train" |
| ], |
| [ |
| "loss", |
| "optim", |
| "train" |
| ], |
| [ |
| "networks", |
| "optim", |
| "train" |
| ], |
| [ |
| "loss", |
| "networks", |
| "optim", |
| "train" |
| ] |
| ], |
| "destructive_control_detected": true, |
| "destructive_control_reversed_success_rates": { |
| "Deepseek-v3.2": [ |
| 0.0, |
| 8.3, |
| 47.2, |
| 75.0 |
| ], |
| "Devstral2": [ |
| 0.0, |
| 0.0, |
| 27.8, |
| 29.2 |
| ], |
| "GPT-OSS-120b": [ |
| 0.0, |
| 8.3, |
| 11.1, |
| 50.0 |
| ] |
| }, |
| "environments_with_higher_two_module_ceiling": 3, |
| "literal_claim": "As the number of editable modules increases in DiscoBench tasks, agent success rates consistently decline while the achievable performance ceiling rises (Appendix G).", |
| "maximum_return_by_module_count": { |
| "1": [ |
| 99.97, |
| 68.07, |
| 39.6, |
| 189.75 |
| ], |
| "2": [ |
| 106.12, |
| 66.16, |
| 69.42, |
| 191.62 |
| ], |
| "3": [ |
| 88.58, |
| 65.2, |
| 64.85, |
| 186.75 |
| ], |
| "4": [ |
| null, |
| null, |
| null, |
| null |
| ] |
| }, |
| "mean_ceiling_increase": 8.9825, |
| "monotonicity_by_model": { |
| "Deepseek-v3.2": true, |
| "Devstral2": true, |
| "GPT-OSS-120b": true |
| }, |
| "single_module_mean_environment_ceiling": 99.3475, |
| "source_configuration_rows": [ |
| { |
| "configuration": "Optimiser", |
| "module_count": 1, |
| "returns": [ |
| 74.97, |
| 62.86, |
| 18.11, |
| 181.25 |
| ], |
| "success_rate": 83.33 |
| }, |
| { |
| "configuration": "Loss", |
| "module_count": 1, |
| "returns": [ |
| 83.91, |
| 62.58, |
| 39.6, |
| 179.5 |
| ], |
| "success_rate": 77.78 |
| }, |
| { |
| "configuration": "Network", |
| "module_count": 1, |
| "returns": [ |
| 99.97, |
| 68.07, |
| 14.2, |
| 189.75 |
| ], |
| "success_rate": 33.33 |
| }, |
| { |
| "configuration": "Train", |
| "module_count": 1, |
| "returns": [ |
| 8.41, |
| 8.41, |
| 3.73, |
| 177.12 |
| ], |
| "success_rate": 11.11 |
| }, |
| { |
| "configuration": "Loss + Optimiser", |
| "module_count": 2, |
| "returns": [ |
| 84.47, |
| 62.89, |
| 20.5, |
| 181.38 |
| ], |
| "success_rate": 61.11 |
| }, |
| { |
| "configuration": "Network + Optimiser", |
| "module_count": 2, |
| "returns": [ |
| 91.44, |
| 65.25, |
| 19.77, |
| 191.62 |
| ], |
| "success_rate": 44.44 |
| }, |
| { |
| "configuration": "Loss + Network", |
| "module_count": 2, |
| "returns": [ |
| 106.12, |
| 66.16, |
| 69.42, |
| 184.0 |
| ], |
| "success_rate": 38.89 |
| }, |
| { |
| "configuration": "Loss + Train", |
| "module_count": 2, |
| "returns": [ |
| 8.56, |
| 61.91, |
| 3.91, |
| 169.38 |
| ], |
| "success_rate": 22.22 |
| }, |
| { |
| "configuration": "Optimiser + Train", |
| "module_count": 2, |
| "returns": [ |
| 34.61, |
| 29.45, |
| null, |
| null |
| ], |
| "success_rate": 5.56 |
| }, |
| { |
| "configuration": "Network + Train", |
| "module_count": 2, |
| "returns": [ |
| null, |
| null, |
| null, |
| null |
| ], |
| "success_rate": 0.0 |
| }, |
| { |
| "configuration": "Loss + Network + Optimiser", |
| "module_count": 3, |
| "returns": [ |
| 88.58, |
| 65.2, |
| 64.85, |
| 186.75 |
| ], |
| "success_rate": 11.11 |
| }, |
| { |
| "configuration": "Loss + Optimiser + Train", |
| "module_count": 3, |
| "returns": [ |
| 0.3, |
| 2.95, |
| null, |
| null |
| ], |
| "success_rate": 11.11 |
| }, |
| { |
| "configuration": "Loss + Network + Train", |
| "module_count": 3, |
| "returns": [ |
| null, |
| null, |
| null, |
| null |
| ], |
| "success_rate": 0.0 |
| }, |
| { |
| "configuration": "Network + Optimiser + Train", |
| "module_count": 3, |
| "returns": [ |
| null, |
| null, |
| null, |
| null |
| ], |
| "success_rate": 0.0 |
| }, |
| { |
| "configuration": "Loss + Network + Optimiser + Train", |
| "module_count": 4, |
| "returns": [ |
| null, |
| null, |
| null, |
| null |
| ], |
| "success_rate": 0.0 |
| } |
| ], |
| "success_rate_by_editable_module_count": { |
| "Deepseek-v3.2": [ |
| 75.0, |
| 47.2, |
| 8.3, |
| 0.0 |
| ], |
| "Devstral2": [ |
| 29.2, |
| 27.8, |
| 0.0, |
| 0.0 |
| ], |
| "GPT-OSS-120b": [ |
| 50.0, |
| 11.1, |
| 8.3, |
| 0.0 |
| ] |
| }, |
| "two_module_mean_environment_ceiling": 108.33 |
| } |
|
|
