File size: 3,246 Bytes
1bc7a81 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 | {
"primary_comparison": "Matched original-architecture GRASP control, A1, A2; training replicate 0; three shared outer splits.",
"matched_endpoint_split_triplets": 69,
"individual_prediction_cells_verified": 207,
"model_mean_endpoint_mae": {
"control": 0.3702320498982857,
"a1": 0.3685879578170505,
"a2": 0.3662318774785543
},
"model_split_macro_population_sd": {
"control": 0.004206375431646905,
"a1": 0.0010903069555724331,
"a2": 0.002152111407523036
},
"lowest_mean_endpoint_counts": {
"a1": 10,
"control": 8,
"a2": 5
},
"contrasts": [
{
"baseline": "control",
"changed": "a1",
"mean_mae_difference": -0.001644092081235165,
"relative_macro_mae_reduction_pct": 0.4440707069220114,
"endpoint_bootstrap_lower95": -0.011490278989452225,
"endpoint_bootstrap_upper95": 0.008019370245648885,
"changed_lower_endpoints": 11,
"changed_higher_endpoints": 12,
"changed_lower_split_cells": 36
},
{
"baseline": "control",
"changed": "a2",
"mean_mae_difference": -0.00400017241973136,
"relative_macro_mae_reduction_pct": 1.0804500639073136,
"endpoint_bootstrap_lower95": -0.012635722835392695,
"endpoint_bootstrap_upper95": 0.004425803942156549,
"changed_lower_endpoints": 14,
"changed_higher_endpoints": 9,
"changed_lower_split_cells": 37
},
{
"baseline": "a1",
"changed": "a2",
"mean_mae_difference": -0.002356080338496194,
"relative_macro_mae_reduction_pct": 0.6392179365951124,
"endpoint_bootstrap_lower95": -0.010505056285555462,
"endpoint_bootstrap_upper95": 0.004859561931627686,
"changed_lower_endpoints": 8,
"changed_higher_endpoints": 15,
"changed_lower_split_cells": 38
}
],
"repeat_context": {
"control": 0.3652608053129453,
"a1": 0.36943776142909285
},
"historical_released_grasp_macro_mae": 0.374183478126786,
"bootstrap": {
"draws": 100000,
"seed": 20261024,
"unit": "endpoint mean across the three fixed splits",
"interval": "95% percentile, exploratory; no multiplicity correction"
},
"checks": "Passed: cohort hashes and row counts, ordered test targets, split IDs, seeds, run configs, software versions, Step2 data/recipe, initialized heads; all primary MAEs recomputed.",
"limitations": [
"One pretraining seed per architecture and one matched downstream replicate for A2.",
"A2 has fewer downstream replicates than the completed control/A1 repeated evaluation.",
"Fixed test molecules recur across outer splits; they are not three independent data collections.",
"Different endpoint scales make the macro MAE a collection-specific score, not a scale-free effect.",
"Endpoint bootstrap omits pretraining-seed variance and does not establish architectural causation; endpoints share data sources.",
"Chemistry-enabled runner hash differs intentionally from control/A1; scientific settings and cohorts were compared explicitly.",
"A2 TDC and A3 downstream results are not covered. No chemical mechanism for individual gains or losses is inferred."
]
}
|