Thibaut Forest Claude Opus 5 (1M context) commited on
Commit ·
4be2283
1
Parent(s): f5f3fda
Phase 57: CV replaced with building-GroupKFold (user directive) - test group planned as standalone fleet, stratified on failures + travel centrality
Browse filesSimulates the held-out deployment path (static foreign fit) the private
split's unseen fleets take. Purge harness retired (three deployment blind
spots, phase 56); recoverable at f5f3fda.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
- EXPERIMENTS.md +19 -0
- batteryswap_example/cv.py +199 -290
EXPERIMENTS.md
CHANGED
|
@@ -3110,3 +3110,22 @@ candidate survives count-matched adjudication. Instruments going forward:
|
|
| 3110 |
deployment-mirror walk-forward (past-only fits) as the public predictor;
|
| 3111 |
building-GroupKFold with test-group-planned-as-fleet (user proposal,
|
| 3112 |
2026-08-22) as the held-out/private-fleet predictor.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3110 |
deployment-mirror walk-forward (past-only fits) as the public predictor;
|
| 3111 |
building-GroupKFold with test-group-planned-as-fleet (user proposal,
|
| 3112 |
2026-08-22) as the held-out/private-fleet predictor.
|
| 3113 |
+
|
| 3114 |
+
## Phase 57 - CV replaced (user directive v3): building-GroupKFold, test group planned as a standalone fleet
|
| 3115 |
+
|
| 3116 |
+
The temporal purge is RETIRED as THE SCORE (its three deployment blind spots
|
| 3117 |
+
are the phase-56 record). New sole harness in cv.py: buildings -> 5 folds,
|
| 3118 |
+
stratified deterministically on failure count (snake assignment; fold deaths
|
| 3119 |
+
22/18/14/15/13 over 5/5/5/5/4 buildings) and travel centrality as the
|
| 3120 |
+
location proxy. Per fold: ONE fit on the TRAIN buildings' closed pairs (full
|
| 3121 |
+
history - deployment-legal for foreign fleets), then for each of the 48
|
| 3122 |
+
scenario times the TEST buildings are planned as a COMPLETE standalone fleet
|
| 3123 |
+
and costed. SCORE = mean over folds of fold-mean cost. This is a measured
|
| 3124 |
+
simulation of submit.py's held-out (static foreign fit) path - the path the
|
| 3125 |
+
private split's unseen fleets will take - with no per-building attribution
|
| 3126 |
+
problem (the fleet IS the test group) and no partial-fleet count bias
|
| 3127 |
+
(phase 26). Fits never see the test buildings; nothing sees the future of a
|
| 3128 |
+
label it trains on beyond what deployment itself would know.
|
| 3129 |
+
CLI: `cv bgkf` / `cv cand <label>` / `cv report`. Purge harness recoverable
|
| 3130 |
+
at f5f3fda; walk-forward family at 2000281. Candidate rule strengthened by
|
| 3131 |
+
phase 56: wins must hold at ~flat swaps PER CELL, not just at the mean.
|
batteryswap_example/cv.py
CHANGED
|
@@ -1,52 +1,46 @@
|
|
| 1 |
-
"""THE CV (user directive 2026-08-22):
|
| 2 |
-
|
| 3 |
-
ONE protocol lives here, deliberately.
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
screen fold comparison, LOBO/walk-forward all48oof variants, cv lobo,
|
| 40 |
-
fixed-N ablation) were REMOVED 2026-08-22 per the same directive - one
|
| 41 |
-
instrument, so nothing else can be read by accident. They live in git
|
| 42 |
-
history (last full version: commit 227c214 and its successors that day).
|
| 43 |
-
exp_x29/exp_x30 referenced the removed ablation helpers and need that
|
| 44 |
-
older cv.py to run.
|
| 45 |
|
| 46 |
Usage
|
| 47 |
-
venv/bin/python -m batteryswap_example.cv
|
| 48 |
-
venv/bin/python -m batteryswap_example.cv
|
| 49 |
-
venv/bin/python -m batteryswap_example.cv
|
| 50 |
"""
|
| 51 |
import sys
|
| 52 |
import time
|
|
@@ -64,22 +58,18 @@ WINDOW_DAYS = int(mc.WINDOW_DAYS)
|
|
| 64 |
# already-mutated mc.FEATURES (a second seed used to append twice)
|
| 65 |
BASE_FEATURES = list(mc.FEATURES)
|
| 66 |
|
| 67 |
-
# The deployed champion
|
| 68 |
-
#
|
| 69 |
-
#
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
CHAMPION = dict(model='logbag_rank', blend=('gbm', 0.30),
|
| 75 |
-
planner='tsroute', p_star=0.8, day_q=0.05,
|
| 76 |
-
ec_margin=0.8, ec_pmax=25.0, split_days=7,
|
| 77 |
-
floor=2.5, floor_gate=20, floor_zone=1)
|
| 78 |
|
| 79 |
BLOCKS = (('warmup s_0-7', lambda o: o <= 7),
|
| 80 |
('panel s_8-41', lambda o: (o >= 8) & (o <= 41)),
|
| 81 |
('nearend s_42-47', lambda o: o >= 42),
|
| 82 |
-
('ALL
|
| 83 |
|
| 84 |
|
| 85 |
def load_table(dataset_path=Path('dataset')):
|
|
@@ -90,9 +80,6 @@ def load_table(dataset_path=Path('dataset')):
|
|
| 90 |
# within-origin fleet ranks: leak-free (data <= origin only)
|
| 91 |
t['r_vlast'] = t.groupby('origin_idx')['v_last'].rank(pct=True)
|
| 92 |
t['r_tau'] = t.groupby('origin_idx')['tau_projected'].rank(pct=True)
|
| 93 |
-
# aux usage-dynamics and Wiener-FHT columns (same merges the old
|
| 94 |
-
# harness __main__ did): cached parquets, data <= origin only; inert
|
| 95 |
-
# unless a variant/model asks for the columns
|
| 96 |
try:
|
| 97 |
from batteryswap_example.aux_features import load as load_aux
|
| 98 |
aux = load_aux(dataset_path)
|
|
@@ -111,21 +98,29 @@ def load_table(dataset_path=Path('dataset')):
|
|
| 111 |
return t
|
| 112 |
|
| 113 |
|
| 114 |
-
def
|
| 115 |
-
"""
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 129 |
|
| 130 |
|
| 131 |
# --------------------------------------------------------------------------
|
|
@@ -136,120 +131,42 @@ def _base(t):
|
|
| 136 |
return BASE_FEATURES
|
| 137 |
|
| 138 |
|
| 139 |
-
def
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
return BASE_FEATURES + ['z_days', 'z_in', 'z_rate']
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
def _vcorr(t):
|
| 148 |
-
return BASE_FEATURES + ['v_corr_last', 'thresh_dist']
|
| 149 |
|
| 150 |
|
| 151 |
def _vrel(t):
|
| 152 |
-
|
| 153 |
-
# scale is not the train fleet's, but its within-fleet ordering is
|
| 154 |
-
return BASE_FEATURES + [c for c in ('vrel', 'vmin7_rank')
|
| 155 |
-
if c in t.columns]
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
def _vrel_only(t):
|
| 159 |
-
return BASE_FEATURES + ['vrel']
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
def _vmin7_only(t):
|
| 163 |
-
return BASE_FEATURES + ['vmin7_rank']
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
def _v_last_only(t):
|
| 167 |
-
# the phase-39 claim under test: two blind sessions found RAW CURRENT
|
| 168 |
-
# VOLTAGE alone beating their champion's ranker at the operating N
|
| 169 |
-
# (.721 vs .668, .703 vs .672). If that reproduces here, our feature set
|
| 170 |
-
# is over-pruned; if it does not, that lineage's ablation was an artifact
|
| 171 |
-
# of its threshold rule and nothing from it should be ported.
|
| 172 |
-
return ['v_last']
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
def _blind4(t):
|
| 176 |
-
# The blind-repo champion's EXACT feature set as a REPLACEMENT ranker
|
| 177 |
-
# (9a969a6a7dfe: last_voltage/vmin7/n_obs/vrel -> our columns). Phase 47:
|
| 178 |
-
# real head-ordering gain at fixed N (static N=8 +27 [+7,+47]) but the
|
| 179 |
-
# planner converts it to ZERO extra catches (290 vs 291 count-matched)
|
| 180 |
-
# and +79/origin cost via cruder day placement. REJECTED. blind3 shows
|
| 181 |
-
# vrel is not load-bearing - the effect is the small set itself.
|
| 182 |
-
return ['v_last', 'v_min_7', 'n_30', 'vrel']
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
def _blind3(t):
|
| 186 |
-
return ['v_last', 'v_min_7', 'n_30']
|
| 187 |
-
|
| 188 |
|
| 189 |
-
def _ranks(t):
|
| 190 |
-
# phase-8 logbag_rank, STRICT form: the within-origin rank transforms
|
| 191 |
-
# REPLACE the two level features (the registry's logbag_rank ADDS them).
|
| 192 |
-
# Ranks are the cross-fleet transfer device; the original lost the old
|
| 193 |
-
# scenario CV by less than that harness's label-luck floor.
|
| 194 |
-
f = [c for c in BASE_FEATURES if c not in ('v_last', 'tau_projected')]
|
| 195 |
-
return f + ['r_vlast', 'r_tau']
|
| 196 |
|
| 197 |
-
|
| 198 |
-
def _vrel_bldg(t):
|
| 199 |
-
# BUILDING-relative voltage: a per-building offset reorders across
|
| 200 |
-
# buildings. Deployment-legal: computed from the target fleet's own
|
| 201 |
-
# telemetry.
|
| 202 |
-
if 'vrel_bldg' not in t.columns:
|
| 203 |
-
dev_b = (pd.read_csv(Path('dataset') / 'train' / 'devices.csv')
|
| 204 |
-
.drop_duplicates('device_id')
|
| 205 |
-
.set_index('device_id')['building_id'])
|
| 206 |
-
b = t['device_id'].map(dev_b)
|
| 207 |
-
t['vrel_bldg'] = (t['v_last'] - t.groupby([t['origin_idx'], b])
|
| 208 |
-
['v_last'].transform('median'))
|
| 209 |
-
return BASE_FEATURES + ['vrel_bldg']
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
VARIANTS = {'base': _base, 'zone': _zone, 'vcorr': _vcorr, 'vrel': _vrel,
|
| 213 |
-
'vrel_only': _vrel_only, 'vmin7_only': _vmin7_only,
|
| 214 |
-
'v_last_only': _v_last_only, 'blind4': _blind4, 'blind3': _blind3,
|
| 215 |
-
'ranks': _ranks, 'vrel_bldg': _vrel_bldg}
|
| 216 |
|
| 217 |
|
| 218 |
# --------------------------------------------------------------------------
|
| 219 |
-
# THE protocol:
|
| 220 |
# --------------------------------------------------------------------------
|
| 221 |
|
| 222 |
-
def
|
| 223 |
-
|
| 224 |
-
"""
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
scenarios train mostly on the future side, near-end scenarios mostly on
|
| 232 |
-
the past side. Cold-start floor off (no warm-start state to gate on).
|
| 233 |
-
|
| 234 |
-
Candidate re-tests (phase 52 addendum sweep): `kind` swaps the model
|
| 235 |
-
family, `variant` swaps the feature set (VARIANTS key), `blend`=(kind2,
|
| 236 |
-
w2) rank-averages a second family in at weight w2 and then maps the
|
| 237 |
-
blended ORDER back onto the primary family's own p42 values - value-
|
| 238 |
-
preserving, so the count/pricing machinery sees the identical p42
|
| 239 |
-
distribution and only WHO holds each rank changes (the phase-51 lesson:
|
| 240 |
-
a post-hoc blend whose beliefs disagree with the count machinery fails
|
| 241 |
-
for that reason alone).
|
| 242 |
"""
|
| 243 |
-
if kind is None and blend is None:
|
| 244 |
-
# the champion arm carries its own blend; an explicit candidate kind
|
| 245 |
-
# replaces the model config outright (no implicit blend inheritance)
|
| 246 |
-
blend = CHAMPION.get('blend')
|
| 247 |
kind = kind or CHAMPION['model']
|
| 248 |
t = load_table(dataset_path)
|
| 249 |
mc.FEATURES = list(VARIANTS[variant](t))
|
| 250 |
ctx = mc.load_context(dataset_path)
|
| 251 |
W = pd.Timedelta(days=WINDOW_DAYS)
|
| 252 |
closed = t[t['origin'] <= t['origin'].max() - W]
|
|
|
|
|
|
|
| 253 |
|
| 254 |
p = mc.plan_tsroute
|
| 255 |
p.alpha, p.p_be, p.floor_day, p.p_conf = CHAMPION['p_star'], 0.0, 0, 0.5
|
|
@@ -257,156 +174,148 @@ def run_all48_oof(origins=range(48), dataset_path=Path('dataset'), kind=None,
|
|
| 257 |
p.split_days = CHAMPION['split_days']
|
| 258 |
p.floor_scale, p.recent_deaths = 0.0, 0.0
|
| 259 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 260 |
recs = []
|
| 261 |
-
for
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
if
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
pred['p42'] = pd.Series(
|
| 298 |
-
vals, index=r.sort_values(ascending=False).index
|
| 299 |
-
).reindex(pred.index)
|
| 300 |
-
plan = mc.plan_tsroute(pred, None, locs, T, CHAMPION['day_q'],
|
| 301 |
-
travel_costs=c['travel_costs'],
|
| 302 |
-
settings=c['settings'])
|
| 303 |
-
_, _, overall = evaluate_plan(plan, locs, c['travel_costs'],
|
| 304 |
-
c['settings'], eol_times=eol,
|
| 305 |
-
start_time=T, verbose=0)
|
| 306 |
-
in_win = plan.loc[plan['day'] <= T + W, 'battery']
|
| 307 |
-
recs.append(dict(overall, n_swaps=int(in_win.nunique()),
|
| 308 |
-
origin=idx, arm='oof'))
|
| 309 |
-
print(f"s_{idx}: oof {recs[-1]['total_cost']:7.1f}",
|
| 310 |
-
flush=True, file=sys.stderr)
|
| 311 |
return pd.DataFrame(recs)
|
| 312 |
|
| 313 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 314 |
# --------------------------------------------------------------------------
|
| 315 |
# candidate sweep on the one harness
|
| 316 |
# --------------------------------------------------------------------------
|
| 317 |
|
| 318 |
-
|
| 319 |
'logistic': dict(kind='logistic'),
|
| 320 |
'rf': dict(kind='rf'),
|
| 321 |
'gbm': dict(kind='gbm'),
|
| 322 |
-
'
|
| 323 |
-
'
|
|
|
|
| 324 |
'vrel': dict(variant='vrel'),
|
| 325 |
-
'vrel_only': dict(variant='vrel_only'),
|
| 326 |
-
'vmin7_only': dict(variant='vmin7_only'),
|
| 327 |
-
'v_last_only': dict(variant='v_last_only'),
|
| 328 |
-
'blind4': dict(variant='blind4'),
|
| 329 |
-
'blind3': dict(variant='blind3'),
|
| 330 |
'rankavg25rf': dict(blend=('rf', 0.25)),
|
| 331 |
'rankavg25gbm': dict(blend=('gbm', 0.25)),
|
| 332 |
-
|
| 333 |
-
# candidates whose rejection came from a self-fit surface while a
|
| 334 |
-
# building-split instrument disagreed or was never consulted
|
| 335 |
-
'logbagC03': dict(kind='logbag_c03'),
|
| 336 |
-
'logbagC01': dict(kind='logbag_c01'),
|
| 337 |
-
'logbag_rank': dict(kind='logbag_rank'),
|
| 338 |
-
'ranks': dict(variant='ranks'),
|
| 339 |
-
'vrel_bldg': dict(variant='vrel_bldg'),
|
| 340 |
-
'logbag_aux': dict(kind='logbag_aux'),
|
| 341 |
-
'aux_ucb05': dict(kind='aux_ucb05'),
|
| 342 |
-
'aux_ucb10': dict(kind='aux_ucb10'),
|
| 343 |
-
'dthaz': dict(kind='dthaz'),
|
| 344 |
-
'rankavg25tau': dict(blend=('tau', 0.25)),
|
| 345 |
-
'rankavg25fht': dict(blend=('wiener', 0.25)),
|
| 346 |
}
|
| 347 |
|
| 348 |
|
| 349 |
-
def
|
| 350 |
-
|
| 351 |
import glob
|
| 352 |
base = pd.read_csv(base_csv)
|
| 353 |
-
b = base[
|
| 354 |
-
print('paired vs champion (logbag/base,
|
| 355 |
print('swaps (plan-derived) shown because a cost delta can be appetite '
|
| 356 |
-
'in disguise
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
f"{
|
| 363 |
out = []
|
| 364 |
for fn in sorted(glob.glob(pattern)):
|
| 365 |
-
d = pd.read_csv(fn).set_index('origin')
|
| 366 |
label = d['cand'].iloc[0]
|
| 367 |
delta = (d['total_cost'] - b['total_cost']).dropna()
|
| 368 |
-
|
| 369 |
-
out.append((delta.mean(), label,
|
| 370 |
-
for _, label,
|
| 371 |
-
|
|
|
|
| 372 |
f"{int((delta < 0).sum()):3d}W/{int((delta > 0).sum()):2d}L "
|
| 373 |
-
f"{d.loc[(d.index >= 8) & (d.index <= 41), 'total_cost'].mean():7.1f} "
|
| 374 |
-
f"{pan.mean():+7.1f} {int((pan < 0).sum()):3d}W/{int((pan > 0).sum()):2d}L "
|
| 375 |
f"{d['n_swaps'].mean():5.1f}")
|
| 376 |
|
| 377 |
|
| 378 |
-
def oof_report(df):
|
| 379 |
-
scorecard(df, 'all48oof / scenario-clean (temporal purge) / floor off')
|
| 380 |
-
# THE SCORE (user, 2026-08-22): one number - the mean cost per scenario
|
| 381 |
-
# over all 48, full-fleet plans, scenario-clean predictions. Blocks
|
| 382 |
-
# above are diagnostics.
|
| 383 |
-
print(f"\nSCORE (scenario-clean temporal purge, mean cost/scenario): "
|
| 384 |
-
f"{df['total_cost'].mean():.1f}")
|
| 385 |
-
|
| 386 |
-
|
| 387 |
def main():
|
| 388 |
-
proto = sys.argv[1] if len(sys.argv) > 1 else '
|
| 389 |
t0 = time.time()
|
| 390 |
-
if proto == '
|
| 391 |
-
|
| 392 |
-
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
df.to_csv('/tmp/cv_all48oof.csv', index=False)
|
| 396 |
-
oof_report(df)
|
| 397 |
-
elif proto == 'oofcand':
|
| 398 |
label = sys.argv[2]
|
| 399 |
-
df =
|
| 400 |
df['cand'] = label
|
| 401 |
-
df.to_csv(f'/tmp/
|
| 402 |
-
|
| 403 |
-
elif proto == '
|
| 404 |
-
|
| 405 |
else:
|
| 406 |
raise SystemExit(
|
| 407 |
-
f'unknown protocol {proto!r} - the
|
| 408 |
-
f'only CV (user directive 2026-08-22).
|
| 409 |
-
f'
|
|
|
|
| 410 |
print(f'\n[{time.time() - t0:.0f}s]')
|
| 411 |
|
| 412 |
|
|
|
|
| 1 |
+
"""THE CV (user directive 2026-08-22 v3): building-GroupKFold cost harness.
|
| 2 |
+
|
| 3 |
+
ONE protocol lives here, deliberately. Buildings are assigned to K folds,
|
| 4 |
+
stratified on battery-failure count and location (travel centrality). For
|
| 5 |
+
each fold: fit ONE model on ALL closed training pairs of the TRAIN buildings
|
| 6 |
+
(their full history - deployment-legal for foreign fleets, whose labels were
|
| 7 |
+
all known when the submission was built), then for each of the 48 scenario
|
| 8 |
+
times plan the TEST buildings as a STANDALONE fleet and evaluate its cost.
|
| 9 |
+
THE SCORE is the mean over folds of the fold's mean scenario cost.
|
| 10 |
+
|
| 11 |
+
Why this protocol (2026-08-22, the day the temporal purge fell):
|
| 12 |
+
|
| 13 |
+
* It simulates the HELD-OUT deployment path: the competition's private split
|
| 14 |
+
contains fleets not in the packaged train table (submit.py's static
|
| 15 |
+
foreign-fit branch exists for exactly this). Train-on-some-buildings /
|
| 16 |
+
plan-the-others-as-a-fleet is that path, measured.
|
| 17 |
+
|
| 18 |
+
* The test group is planned as a COMPLETE fleet, so there is no per-building
|
| 19 |
+
cost-attribution problem (the old LOBO objection) and no partial-fleet
|
| 20 |
+
count bias (phase 26): the count rule operates on the test fleet's own
|
| 21 |
+
sum(p42), exactly as deployment does on a held-out fleet.
|
| 22 |
+
|
| 23 |
+
* The retired temporal purge was refuted as a deployment predictor three
|
| 24 |
+
ways in one day (phase 56): count-coupled changes (purge loved alpha 1.5,
|
| 25 |
+
public +527), label-process exploits (cert-first re-derived at its exact
|
| 26 |
+
pre-refutation signature), and era-hazard calibration leak through
|
| 27 |
+
future-side fits (the frontier: purge -341.7 -> public +215.6; the gain
|
| 28 |
+
was a hidden per-scenario appetite dial, corr(count delta, cost gain)
|
| 29 |
+
0.76). The common thread: its fits saw the future. This harness's fits
|
| 30 |
+
never see the test buildings at all.
|
| 31 |
+
|
| 32 |
+
* Diagnostics are PLAN-DERIVED only (cost blocks + in-window swap count), so
|
| 33 |
+
ANY method that yields a plan can be scored. Candidates must win at ~flat
|
| 34 |
+
swaps; count-coupled wins are untrustworthy (public evidence, 4x).
|
| 35 |
+
|
| 36 |
+
Superseded protocols live in git: the temporal-purge harness (sole validator
|
| 37 |
+
2026-08-22, retired same day) at f5f3fda; the earlier walk-forward/static/
|
| 38 |
+
screen/LOBO family at 2000281.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
|
| 40 |
Usage
|
| 41 |
+
venv/bin/python -m batteryswap_example.cv bgkf # THE SCORE
|
| 42 |
+
venv/bin/python -m batteryswap_example.cv cand <label>
|
| 43 |
+
venv/bin/python -m batteryswap_example.cv report
|
| 44 |
"""
|
| 45 |
import sys
|
| 46 |
import time
|
|
|
|
| 58 |
# already-mutated mc.FEATURES (a second seed used to append twice)
|
| 59 |
BASE_FEATURES = list(mc.FEATURES)
|
| 60 |
|
| 61 |
+
# The deployed champion (submit.py default after phase 56 reverted the
|
| 62 |
+
# frontier): plain logbag / tsroute alpha 0.8 + break-even top-up, censor-
|
| 63 |
+
# aware pricing. Frontier dials exist in submit.py for explicit A/B only.
|
| 64 |
+
CHAMPION = dict(model='logbag', planner='tsroute', p_star=0.8, day_q=0.05,
|
| 65 |
+
ec_margin=0.8, ec_pmax=25.0, split_days=7)
|
| 66 |
+
|
| 67 |
+
N_FOLDS = 5
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
|
| 69 |
BLOCKS = (('warmup s_0-7', lambda o: o <= 7),
|
| 70 |
('panel s_8-41', lambda o: (o >= 8) & (o <= 41)),
|
| 71 |
('nearend s_42-47', lambda o: o >= 42),
|
| 72 |
+
('ALL', lambda o: o >= 0))
|
| 73 |
|
| 74 |
|
| 75 |
def load_table(dataset_path=Path('dataset')):
|
|
|
|
| 80 |
# within-origin fleet ranks: leak-free (data <= origin only)
|
| 81 |
t['r_vlast'] = t.groupby('origin_idx')['v_last'].rank(pct=True)
|
| 82 |
t['r_tau'] = t.groupby('origin_idx')['tau_projected'].rank(pct=True)
|
|
|
|
|
|
|
|
|
|
| 83 |
try:
|
| 84 |
from batteryswap_example.aux_features import load as load_aux
|
| 85 |
aux = load_aux(dataset_path)
|
|
|
|
| 98 |
return t
|
| 99 |
|
| 100 |
|
| 101 |
+
def build_folds(t, ctx, dataset_path=Path('dataset'), k=N_FOLDS):
|
| 102 |
+
"""Deterministic stratified building->fold assignment.
|
| 103 |
+
|
| 104 |
+
Stratified on (1) battery-failure count - unique dying devices per
|
| 105 |
+
building - and (2) location, proxied by travel centrality (mean hours to
|
| 106 |
+
every other building). Buildings are sorted by (deaths desc, centrality,
|
| 107 |
+
id) and snake-assigned to k folds, which balances the lumpy death counts
|
| 108 |
+
(a test fold with ~0 deaths measures nothing) while spreading central
|
| 109 |
+
and remote buildings across folds. No randomness: reruns are identical.
|
| 110 |
+
"""
|
| 111 |
+
dev_b = (pd.read_csv(dataset_path / 'train' / 'devices.csv')
|
| 112 |
+
.drop_duplicates('device_id').set_index('device_id')['building_id'])
|
| 113 |
+
b = t['device_id'].map(dev_b)
|
| 114 |
+
deaths = (t.loc[t['dies42'] == 1].groupby(b[t['dies42'] == 1])['device_id']
|
| 115 |
+
.nunique())
|
| 116 |
+
tc = ctx[0]['travel_costs']
|
| 117 |
+
central = tc[tc['from'] != tc['to']].groupby('from')['hours'].mean()
|
| 118 |
+
rows = pd.DataFrame({'deaths': deaths, 'central': central}).fillna(0)
|
| 119 |
+
rows = rows.sort_values(['deaths', 'central'],
|
| 120 |
+
ascending=[False, True]).reset_index()
|
| 121 |
+
order = list(range(k)) + list(range(k - 1, -1, -1)) # snake
|
| 122 |
+
rows['fold'] = [order[i % (2 * k)] for i in range(len(rows))]
|
| 123 |
+
return rows.set_index('index')['fold'], dev_b
|
| 124 |
|
| 125 |
|
| 126 |
# --------------------------------------------------------------------------
|
|
|
|
| 131 |
return BASE_FEATURES
|
| 132 |
|
| 133 |
|
| 134 |
+
def _ranks_add(t):
|
| 135 |
+
# phase-56 rank features. NOTE: under this harness the prediction-time
|
| 136 |
+
# rank is over the TEST fleet only (a few buildings), which is also what
|
| 137 |
+
# a held-out deployment would compute.
|
| 138 |
+
return BASE_FEATURES + ['r_vlast', 'r_tau']
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 139 |
|
| 140 |
|
| 141 |
def _vrel(t):
|
| 142 |
+
return BASE_FEATURES + [c for c in ('vrel', 'vmin7_rank') if c in t.columns]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 143 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
|
| 145 |
+
VARIANTS = {'base': _base, 'ranks_add': _ranks_add, 'vrel': _vrel}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 146 |
|
| 147 |
|
| 148 |
# --------------------------------------------------------------------------
|
| 149 |
+
# THE protocol: building-GroupKFold, test group planned as a standalone fleet
|
| 150 |
# --------------------------------------------------------------------------
|
| 151 |
|
| 152 |
+
def run_bgkf(dataset_path=Path('dataset'), kind=None, variant='base',
|
| 153 |
+
blend=None, n_folds=N_FOLDS, origins=range(48)):
|
| 154 |
+
"""THE SCORE. For each fold: one fit on the TRAIN buildings' closed pairs
|
| 155 |
+
(full history), then per scenario time plan the TEST buildings as a
|
| 156 |
+
standalone fleet and evaluate. Floor off (harness minimalism; the
|
| 157 |
+
deployed floor is a submit.py concern). Candidate hooks: `kind` swaps the
|
| 158 |
+
model family (mc.MODELS key or SklearnRUL spec), `variant` the feature
|
| 159 |
+
set, `blend`=(kind2, w2) a value-preserving rank blend (order changes,
|
| 160 |
+
the primary's p42 values and hence the count machinery do not).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 161 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 162 |
kind = kind or CHAMPION['model']
|
| 163 |
t = load_table(dataset_path)
|
| 164 |
mc.FEATURES = list(VARIANTS[variant](t))
|
| 165 |
ctx = mc.load_context(dataset_path)
|
| 166 |
W = pd.Timedelta(days=WINDOW_DAYS)
|
| 167 |
closed = t[t['origin'] <= t['origin'].max() - W]
|
| 168 |
+
folds, dev_b = build_folds(t, ctx, dataset_path, k=n_folds)
|
| 169 |
+
t_bldg = t['device_id'].map(dev_b)
|
| 170 |
|
| 171 |
p = mc.plan_tsroute
|
| 172 |
p.alpha, p.p_be, p.floor_day, p.p_conf = CHAMPION['p_star'], 0.0, 0, 0.5
|
|
|
|
| 174 |
p.split_days = CHAMPION['split_days']
|
| 175 |
p.floor_scale, p.recent_deaths = 0.0, 0.0
|
| 176 |
|
| 177 |
+
def _fit(k_, train):
|
| 178 |
+
hazard = k_ in ('aft', 'dthaz', 'dthaz_aux', 'logbag_open',
|
| 179 |
+
'blend_dh', 'blend_dhaux')
|
| 180 |
+
if hazard:
|
| 181 |
+
pool = mc.censored_pairs(t, t['origin'].max())
|
| 182 |
+
keep = set(train['device_id'])
|
| 183 |
+
train = pool[pool['device_id'].isin(keep)]
|
| 184 |
+
make = ((lambda: mc.MODELS[k_]()) if k_ in mc.MODELS
|
| 185 |
+
else (lambda: mc.SklearnRUL(k_)))
|
| 186 |
+
return make().fit(train)
|
| 187 |
+
|
| 188 |
recs = []
|
| 189 |
+
for f in range(n_folds):
|
| 190 |
+
test_b = set(folds.index[folds == f])
|
| 191 |
+
train = closed[~t_bldg.loc[closed.index].isin(test_b)]
|
| 192 |
+
model = _fit(kind, train)
|
| 193 |
+
partner = _fit(blend[0], train) if blend is not None else None
|
| 194 |
+
for idx in origins:
|
| 195 |
+
c = ctx[idx]
|
| 196 |
+
T = c['start_time']
|
| 197 |
+
fleet = c['locs'][c['locs']['building'].isin(test_b)]
|
| 198 |
+
rows = (t[t['origin_idx'] == idx].set_index('device_id')
|
| 199 |
+
.reindex(fleet['battery']).dropna(subset=['v_last']))
|
| 200 |
+
if rows.empty:
|
| 201 |
+
continue
|
| 202 |
+
locs = fleet[fleet['battery'].isin(rows.index)]
|
| 203 |
+
eol = c['eol'].reindex(rows.index)
|
| 204 |
+
pred = model.predict(rows)
|
| 205 |
+
if partner is not None:
|
| 206 |
+
w2 = blend[1]
|
| 207 |
+
r = ((1 - w2) * pred['p42'].rank(pct=True)
|
| 208 |
+
+ w2 * partner.predict(rows)['p42'].rank(pct=True))
|
| 209 |
+
vals = np.sort(pred['p42'].values)[::-1]
|
| 210 |
+
pred = pred.copy()
|
| 211 |
+
pred['p42'] = pd.Series(
|
| 212 |
+
vals, index=r.sort_values(ascending=False).index
|
| 213 |
+
).reindex(pred.index)
|
| 214 |
+
plan = mc.plan_tsroute(pred, None, locs, T, CHAMPION['day_q'],
|
| 215 |
+
travel_costs=c['travel_costs'],
|
| 216 |
+
settings=c['settings'])
|
| 217 |
+
_, _, overall = evaluate_plan(plan, locs, c['travel_costs'],
|
| 218 |
+
c['settings'], eol_times=eol,
|
| 219 |
+
start_time=T, verbose=0)
|
| 220 |
+
in_win = plan.loc[plan['day'] <= T + W, 'battery']
|
| 221 |
+
recs.append(dict(overall, n_swaps=int(in_win.nunique()),
|
| 222 |
+
origin=idx, fold=f, arm='bgkf'))
|
| 223 |
+
print(f"f{f} s_{idx}: {recs[-1]['total_cost']:7.1f}",
|
| 224 |
+
flush=True, file=sys.stderr)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 225 |
return pd.DataFrame(recs)
|
| 226 |
|
| 227 |
|
| 228 |
+
def report(df):
|
| 229 |
+
"""Fold table + scenario-block scorecard + THE SCORE (single value)."""
|
| 230 |
+
print('\n=== building-GroupKFold / test group planned as a fleet / '
|
| 231 |
+
'floor off ===')
|
| 232 |
+
print(f"{'fold':>4s} {'n':>3s} {'total':>8s} {'late':>7s} {'early':>6s} "
|
| 233 |
+
f"{'swaps':>5s}")
|
| 234 |
+
for f, d in df.groupby('fold'):
|
| 235 |
+
print(f"{f:4d} {len(d):3d} {d['total_cost'].mean():8.1f} "
|
| 236 |
+
f"{d['late_swap'].mean():7.1f} {d['early_swap'].mean():6.1f} "
|
| 237 |
+
f"{d['n_swaps'].mean():5.1f}")
|
| 238 |
+
print(f"\n{'block':16s} {'n':>3s} {'total':>8s} {'late':>7s} "
|
| 239 |
+
f"{'early':>6s} {'swaps':>5s}")
|
| 240 |
+
for name, sel in BLOCKS:
|
| 241 |
+
d = df[sel(df['origin'])]
|
| 242 |
+
if d.empty:
|
| 243 |
+
continue
|
| 244 |
+
print(f"{name:16s} {len(d):3d} {d['total_cost'].mean():8.1f} "
|
| 245 |
+
f"{d['late_swap'].mean():7.1f} {d['early_swap'].mean():6.1f} "
|
| 246 |
+
f"{d['n_swaps'].mean():5.1f}")
|
| 247 |
+
score = df.groupby('fold')['total_cost'].mean().mean()
|
| 248 |
+
print(f"\nSCORE (building-GroupKFold, mean over folds of fold-mean "
|
| 249 |
+
f"cost): {score:.1f}")
|
| 250 |
+
|
| 251 |
+
|
| 252 |
# --------------------------------------------------------------------------
|
| 253 |
# candidate sweep on the one harness
|
| 254 |
# --------------------------------------------------------------------------
|
| 255 |
|
| 256 |
+
CANDIDATES = {
|
| 257 |
'logistic': dict(kind='logistic'),
|
| 258 |
'rf': dict(kind='rf'),
|
| 259 |
'gbm': dict(kind='gbm'),
|
| 260 |
+
'dthaz': dict(kind='dthaz'),
|
| 261 |
+
'logbag_rank': dict(kind='logbag_rank'),
|
| 262 |
+
'ranks_add': dict(variant='ranks_add'),
|
| 263 |
'vrel': dict(variant='vrel'),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 264 |
'rankavg25rf': dict(blend=('rf', 0.25)),
|
| 265 |
'rankavg25gbm': dict(blend=('gbm', 0.25)),
|
| 266 |
+
'frontier': dict(kind='logbag_rank', blend=('gbm', 0.30)),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 267 |
}
|
| 268 |
|
| 269 |
|
| 270 |
+
def candidate_report(base_csv='/tmp/cv_bgkf.csv',
|
| 271 |
+
pattern='/tmp/cv_bgkfcand_*.csv'):
|
| 272 |
import glob
|
| 273 |
base = pd.read_csv(base_csv)
|
| 274 |
+
b = base.set_index(['fold', 'origin'])
|
| 275 |
+
print('paired vs champion (logbag/base, building-GroupKFold, floor off).')
|
| 276 |
print('swaps (plan-derived) shown because a cost delta can be appetite '
|
| 277 |
+
'in disguise: trust a win only if swaps is ~flat, and check the '
|
| 278 |
+
'PER-CELL count swing, not just the mean (phase 56 lesson).\n')
|
| 279 |
+
print(f"{'candidate':14s} {'SCORE':>7s} {'delta':>7s} {'SE':>6s} "
|
| 280 |
+
f"{'W/L':>7s} {'swaps':>5s}")
|
| 281 |
+
bs = b.groupby('fold')['total_cost'].mean().mean()
|
| 282 |
+
print(f"{'base':14s} {bs:7.1f} {'-':>7s} {'-':>6s} {'-':>7s} "
|
| 283 |
+
f"{b['n_swaps'].mean():5.1f}")
|
| 284 |
out = []
|
| 285 |
for fn in sorted(glob.glob(pattern)):
|
| 286 |
+
d = pd.read_csv(fn).set_index(['fold', 'origin'])
|
| 287 |
label = d['cand'].iloc[0]
|
| 288 |
delta = (d['total_cost'] - b['total_cost']).dropna()
|
| 289 |
+
score = d.groupby('fold')['total_cost'].mean().mean()
|
| 290 |
+
out.append((delta.mean(), label, score, delta, d))
|
| 291 |
+
for _, label, score, delta, d in sorted(out):
|
| 292 |
+
se = delta.std() / np.sqrt(len(delta))
|
| 293 |
+
print(f"{label:14s} {score:7.1f} {delta.mean():+7.1f} {se:6.1f} "
|
| 294 |
f"{int((delta < 0).sum()):3d}W/{int((delta > 0).sum()):2d}L "
|
|
|
|
|
|
|
| 295 |
f"{d['n_swaps'].mean():5.1f}")
|
| 296 |
|
| 297 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 298 |
def main():
|
| 299 |
+
proto = sys.argv[1] if len(sys.argv) > 1 else 'bgkf'
|
| 300 |
t0 = time.time()
|
| 301 |
+
if proto == 'bgkf':
|
| 302 |
+
df = run_bgkf()
|
| 303 |
+
df.to_csv('/tmp/cv_bgkf.csv', index=False)
|
| 304 |
+
report(df)
|
| 305 |
+
elif proto == 'cand':
|
|
|
|
|
|
|
|
|
|
| 306 |
label = sys.argv[2]
|
| 307 |
+
df = run_bgkf(**CANDIDATES[label])
|
| 308 |
df['cand'] = label
|
| 309 |
+
df.to_csv(f'/tmp/cv_bgkfcand_{label}.csv', index=False)
|
| 310 |
+
report(df)
|
| 311 |
+
elif proto == 'report':
|
| 312 |
+
candidate_report()
|
| 313 |
else:
|
| 314 |
raise SystemExit(
|
| 315 |
+
f'unknown protocol {proto!r} - the building-GroupKFold harness '
|
| 316 |
+
f'is the only CV (user directive 2026-08-22 v3). The temporal '
|
| 317 |
+
f'purge lives in git at f5f3fda; the older walk-forward family '
|
| 318 |
+
f'at 2000281.')
|
| 319 |
print(f'\n[{time.time() - t0:.0f}s]')
|
| 320 |
|
| 321 |
|