Submission 004: generalization-fixed v3 planner

scripts/train_submission.py

@@ -32,6 +32,11 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--quantile", type=float, default=0.05)
     parser.add_argument("--event-risk-threshold", type=float, default=0.50)
     parser.add_argument("--prediction-offset-days", type=float, default=-5.0)
+    parser.add_argument("--risk-calibration-scale", type=float, default=1.5)
+    parser.add_argument("--expected-gain-margin", type=float, default=10.0)
+    parser.add_argument("--expected-service-cost-hours", type=float, default=2.0)
+    parser.add_argument("--minimum-scheduled-batteries", type=int, default=16)
+    parser.add_argument("--maximum-scheduled-batteries", type=int, default=20)
     parser.add_argument("--calibration-folds", type=int, default=5)
     parser.add_argument("--skip-evaluation", action="store_true")
     return parser.parse_args()
@@ -65,6 +70,11 @@ def main() -> None:
             event_risk_threshold=args.event_risk_threshold,
             prediction_offset_days=args.prediction_offset_days,
             use_expected_cost=True,
+            risk_calibration_scale=args.risk_calibration_scale,
+            expected_gain_margin=args.expected_gain_margin,
+            expected_service_cost_hours=args.expected_service_cost_hours,
+            minimum_scheduled_batteries=args.minimum_scheduled_batteries,
+            maximum_scheduled_batteries=args.maximum_scheduled_batteries,
             capacity_lookahead_days=21,
         ),
     )
@@ -77,12 +87,19 @@ def main() -> None:
         "quantile": args.quantile,
         "event_risk_threshold": args.event_risk_threshold,
         "prediction_offset_days": args.prediction_offset_days,
+        "risk_calibration_scale": args.risk_calibration_scale,
+        "expected_gain_margin": args.expected_gain_margin,
+        "expected_service_cost_hours": args.expected_service_cost_hours,
+        "minimum_scheduled_batteries": args.minimum_scheduled_batteries,
+        "maximum_scheduled_batteries": args.maximum_scheduled_batteries,
         "calibration_folds": args.calibration_folds,
         "planner_policy": {
             "expected_cost": True,
             "risk_calibration_scale": planner.policy.risk_calibration_scale,
             "expected_service_cost_hours": planner.policy.expected_service_cost_hours,
             "expected_gain_margin": planner.policy.expected_gain_margin,
+            "minimum_scheduled_batteries": planner.policy.minimum_scheduled_batteries,
+            "maximum_scheduled_batteries": planner.policy.maximum_scheduled_batteries,
             "prediction_offset_days": planner.policy.prediction_offset_days,
             "stale_risk_cutoff_days": planner.policy.stale_risk_cutoff_days,
             "recent_gap_risk_factor": planner.policy.recent_gap_risk_factor,

scripts/validate_submission.py

@@ -25,16 +25,47 @@ def _numbers(value: str) -> list[float]:
     return [float(item.strip()) for item in value.split(",") if item.strip()]
 
 
+def _integers(value: str) -> list[int]:
+    return [int(item.strip()) for item in value.split(",") if item.strip()]
+
+
+def _schedule_bands(quotas: str, bands: str) -> list[tuple[int, int]]:
+    values = [(quota, quota) for quota in _integers(quotas)]
+    for item in bands.split(","):
+        item = item.strip()
+        if not item:
+            continue
+        minimum, maximum = item.split(":", maxsplit=1)
+        values.append((int(minimum), int(maximum)))
+    return list(dict.fromkeys(values))
+
+
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser()
     parser.add_argument("--dataset-path", type=Path, default=Path("data/raw/train"))
     parser.add_argument("--folds", type=int, default=5)
     parser.add_argument("--quantile", type=float, default=0.05)
     parser.add_argument("--inner-calibration-folds", type=int, default=3)
+    parser.add_argument(
+        "--outer-group",
+        choices=("battery", "building"),
+        default="building",
+        help=(
+            "Hold out entire buildings by default to emulate hidden-domain "
+            "generalization; use battery only to reproduce legacy reports."
+        ),
+    )
     parser.add_argument("--risk-scales", default="1.50")
     parser.add_argument("--gain-margins", default="10")
     parser.add_argument("--service-costs", default="2")
     parser.add_argument("--offsets", default="-5")
+    parser.add_argument("--schedule-quotas", default="")
+    parser.add_argument("--schedule-bands", default="16:20")
+    parser.add_argument(
+        "--predictions-csv",
+        type=Path,
+        help="Reuse previously generated OOF predictions for policy-only sweeps.",
+    )
     parser.add_argument(
         "--output", type=Path, default=Path("artifacts/submission_cv_final.json")
     )
@@ -63,21 +94,30 @@ def main() -> None:
 
     oof_risk = np.full(len(training), np.nan)
     oof_rul = np.full(len(training), np.nan)
-    splitter = GroupKFold(n_splits=args.folds)
-    groups = training["battery"].astype(str)
-    for fold_number, (train_index, valid_index) in enumerate(
-        splitter.split(training, groups=groups), start=1
-    ):
-        model = fit_event_time_model(
-            training.iloc[train_index],
-            quantile=args.quantile,
-            dataset_revision=DATASET_REVISION,
-            random_state=2026 + fold_number,
-            calibration_folds=args.inner_calibration_folds,
-        )
-        valid = training.iloc[valid_index]
-        oof_risk[valid_index] = model.predict_event_risk(valid)
-        oof_rul[valid_index] = model.predict_rul(valid)
+    if args.predictions_csv is not None:
+        cached = pd.read_csv(args.predictions_csv).set_index(["scenario", "battery"])
+        keys = pd.MultiIndex.from_frame(training[["scenario", "battery"]])
+        aligned = cached.reindex(keys)
+        if aligned[["oof_event_risk", "oof_rul_days"]].isna().any().any():
+            raise ValueError("Cached OOF predictions do not cover the validation rows")
+        oof_risk[:] = aligned["oof_event_risk"].to_numpy(dtype=float)
+        oof_rul[:] = aligned["oof_rul_days"].to_numpy(dtype=float)
+    else:
+        splitter = GroupKFold(n_splits=args.folds)
+        groups = training[args.outer_group].astype(str)
+        for fold_number, (train_index, valid_index) in enumerate(
+            splitter.split(training, groups=groups), start=1
+        ):
+            model = fit_event_time_model(
+                training.iloc[train_index],
+                quantile=args.quantile,
+                dataset_revision=DATASET_REVISION,
+                random_state=2026 + fold_number,
+                calibration_folds=args.inner_calibration_folds,
+            )
+            valid = training.iloc[valid_index]
+            oof_risk[valid_index] = model.predict_event_risk(valid)
+            oof_rul[valid_index] = model.predict_rul(valid)
 
     due = (
         training["event_observed"].astype(bool)
@@ -114,76 +154,84 @@ def main() -> None:
     classification["due_within_horizon_rul_mae_days"] = float(np.mean(due_rul_error))
 
     policy_results = []
-    placeholder_model = model
+    placeholder_model = None
     for risk_scale in _numbers(args.risk_scales):
         for gain_margin in _numbers(args.gain_margins):
             for service_cost in _numbers(args.service_costs):
                 for offset in _numbers(args.offsets):
-                    planner = CompetitionPlanner(
-                        placeholder_model,
-                        PlannerPolicy(
-                            event_risk_threshold=0.50,
-                            prediction_offset_days=offset,
-                            use_expected_cost=True,
-                            risk_calibration_scale=risk_scale,
-                            expected_service_cost_hours=service_cost,
-                            expected_gain_margin=gain_margin,
-                            capacity_lookahead_days=21,
-                        ),
-                    )
-                    scores = []
-                    scheduled_counts = []
-                    for scenario_name, (scenario, locs, not_dead) in scenario_inputs.items():
-                        mask = training["scenario"].eq(scenario_name).to_numpy()
-                        snapshot = training.loc[mask]
-                        risk = oof_risk[mask]
-                        rul = oof_rul[mask]
-                        plan = planner.plan_snapshot(
-                            snapshot,
-                            locs,
-                            scenario["travel_costs"],
-                            scenario["settings"],
-                            scenario["start_time"],
-                            predicted_rul=rul,
-                            predicted_risk=risk,
-                        )
-                        start = pd.Timestamp(scenario["start_time"])
-                        horizon_end = start + pd.Timedelta(
-                            days=scenario["settings"].planning_window_days
+                    for minimum, maximum in _schedule_bands(
+                        args.schedule_quotas, args.schedule_bands
+                    ):
+                        planner = CompetitionPlanner(
+                            placeholder_model,
+                            PlannerPolicy(
+                                event_risk_threshold=0.50,
+                                prediction_offset_days=offset,
+                                use_expected_cost=True,
+                                risk_calibration_scale=risk_scale,
+                                expected_service_cost_hours=service_cost,
+                                expected_gain_margin=gain_margin,
+                                capacity_lookahead_days=21,
+                                minimum_scheduled_batteries=minimum,
+                                maximum_scheduled_batteries=maximum,
+                            ),
                         )
-                        scheduled_counts.append(int(plan["day"].le(horizon_end).sum()))
-                        _, _, score = evaluate_plan(
-                            plan,
-                            locs,
-                            scenario["travel_costs"],
-                            scenario["settings"],
-                            eol_times=not_dead,
-                            start_time=start,
-                            verbose=0,
+                        scores = []
+                        scheduled_counts = []
+                        for scenario_name, (scenario, locs, not_dead) in scenario_inputs.items():
+                            mask = training["scenario"].eq(scenario_name).to_numpy()
+                            snapshot = training.loc[mask]
+                            risk = oof_risk[mask]
+                            rul = oof_rul[mask]
+                            plan = planner.plan_snapshot(
+                                snapshot,
+                                locs,
+                                scenario["travel_costs"],
+                                scenario["settings"],
+                                scenario["start_time"],
+                                predicted_rul=rul,
+                                predicted_risk=risk,
+                            )
+                            start = pd.Timestamp(scenario["start_time"])
+                            horizon_end = start + pd.Timedelta(
+                                days=scenario["settings"].planning_window_days
+                            )
+                            scheduled_counts.append(int(plan["day"].le(horizon_end).sum()))
+                            _, _, score = evaluate_plan(
+                                plan,
+                                locs,
+                                scenario["travel_costs"],
+                                scenario["settings"],
+                                eol_times=not_dead,
+                                start_time=start,
+                                verbose=0,
+                            )
+                            scores.append(score)
+                        mean_score = pd.concat(scores, axis=1).mean(axis=1)
+                        policy_results.append(
+                            {
+                                "risk_calibration_scale": risk_scale,
+                                "expected_gain_margin": gain_margin,
+                                "expected_service_cost_hours": service_cost,
+                                "prediction_offset_days": offset,
+                                "minimum_scheduled_batteries": minimum,
+                                "maximum_scheduled_batteries": maximum,
+                                "stale_risk_cutoff_days": planner.policy.stale_risk_cutoff_days,
+                                "recent_gap_risk_factor": planner.policy.recent_gap_risk_factor,
+                                "stale_risk_factor": planner.policy.stale_risk_factor,
+                                "building_batch_window_days": planner.policy.building_batch_window_days,
+                                "capacity_operational_cost_weight": planner.policy.capacity_operational_cost_weight,
+                                "mean_scheduled_batteries": float(np.mean(scheduled_counts)),
+                                **{key: float(value) for key, value in mean_score.items()},
+                            }
                         )
-                        scores.append(score)
-                    mean_score = pd.concat(scores, axis=1).mean(axis=1)
-                    policy_results.append(
-                        {
-                            "risk_calibration_scale": risk_scale,
-                            "expected_gain_margin": gain_margin,
-                            "expected_service_cost_hours": service_cost,
-                            "prediction_offset_days": offset,
-                            "stale_risk_cutoff_days": planner.policy.stale_risk_cutoff_days,
-                            "recent_gap_risk_factor": planner.policy.recent_gap_risk_factor,
-                            "stale_risk_factor": planner.policy.stale_risk_factor,
-                            "building_batch_window_days": planner.policy.building_batch_window_days,
-                            "capacity_operational_cost_weight": planner.policy.capacity_operational_cost_weight,
-                            "mean_scheduled_batteries": float(np.mean(scheduled_counts)),
-                            **{key: float(value) for key, value in mean_score.items()},
-                        }
-                    )
 
     policy_results.sort(key=lambda item: item["total_cost"])
     report = {
         "dataset_revision": DATASET_REVISION,
         "folds": args.folds,
         "inner_calibration_folds": args.inner_calibration_folds,
+        "outer_group": args.outer_group,
         "quantile": args.quantile,
         "classification": classification,
         "policies": policy_results,

src/batteryswapai/competition_features.py

@@ -20,6 +20,12 @@ NON_FEATURE_COLUMNS = {
     "effective_eol",
     "target_rul_days",
     "event_observed",
+    # Dataset/scenario boundary fields are useful to the planner and reports,
+    # but they are not battery-health measurements. Learning from them makes
+    # same-scenario validation optimistic and breaks on a shifted hidden split.
+    "censor_proxy_rul_days",
+    "scenario_month_sin",
+    "scenario_month_cos",
 }
 
 

src/batteryswapai/competition_planner.py

@@ -31,6 +31,8 @@ class PlannerPolicy:
     stale_risk_cutoff_days: float = 7.0
     recent_gap_risk_factor: float = 0.75
     stale_risk_factor: float = 0.10
+    minimum_scheduled_batteries: int = 0
+    maximum_scheduled_batteries: int | None = None
 
 
 class CompetitionPlanner(Planner):
@@ -90,16 +92,6 @@ class CompetitionPlanner(Planner):
         work["predicted_risk"] = risks
         work["target_day"] = np.floor(np.clip(predictions, 0.0, horizon + 1.0)).astype(int)
         if self.policy.use_expected_cost:
-            if "censor_proxy_rul_days" in snapshot.columns:
-                proxy_rul = pd.to_numeric(
-                    snapshot["censor_proxy_rul_days"], errors="coerce"
-                ).to_numpy(dtype=float)
-            else:
-                proxy_rul = (
-                    pd.to_datetime(snapshot["end_time"], errors="coerce").dt.normalize()
-                    + pd.to_timedelta(float(settings.unobserved_eol_days), unit="D")
-                    - start
-                ).dt.total_seconds().to_numpy() / 86400.0
             planned_day = np.clip(predictions, 0.0, float(horizon))
             expected_event_day = np.clip(
                 raw_predictions + self.policy.expected_event_day_shift,
@@ -123,9 +115,12 @@ class CompetitionPlanner(Planner):
                 * np.maximum(emergency_day - expected_event_day, 0.0)
                 + self.policy.expected_service_cost_hours
             )
-            healthy_early_cost = float(settings.early_replacement_penalty_daily) * np.maximum(
-                proxy_rul - planned_day,
-                0.0,
+            # Conditional on no event within the planning horizon, EOL is
+            # later than the horizon. Location end_time is only a dataset
+            # boundary and shifted between train and the hidden submission.
+            healthy_early_cost = (
+                float(settings.early_replacement_penalty_daily)
+                * np.maximum(float(horizon) - planned_day, 0.0)
             )
             expected_gain = (
                 calibrated_risk * (no_schedule_event_cost - scheduled_event_cost)
@@ -142,6 +137,26 @@ class CompetitionPlanner(Planner):
                 (risks >= self.policy.event_risk_threshold) & (predictions <= horizon)
             )
 
+        # Absolute probabilities drifted sharply between train and the public
+        # split. Keep the workload stable and use the model primarily as a
+        # ranking function, which is substantially more robust to calibration
+        # shift. Thresholding still decides the set when it lies in the band.
+        priority_column = "expected_gain" if self.policy.use_expected_cost else "predicted_risk"
+        ranked = work.sort_values(
+            [priority_column, "predicted_risk", "predicted_rul", "battery"],
+            ascending=[False, False, True, True],
+        )
+        minimum = min(max(int(self.policy.minimum_scheduled_batteries), 0), len(work))
+        maximum = self.policy.maximum_scheduled_batteries
+        selected = set(work.index[work["scheduled"]])
+        if len(selected) < minimum:
+            selected.update(ranked.index[:minimum])
+        if maximum is not None and len(selected) > int(maximum):
+            selected = set(
+                ranked.loc[ranked.index.isin(selected)].index[: int(maximum)]
+            )
+        work["scheduled"] = work.index.isin(selected)
+
         candidates = work[work["scheduled"]].copy()
         if not candidates.empty:
             candidates = self._schedule_candidates(

submission_artifacts/planner.joblib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75ed285b5e64b69543d58dc800bf50f508cdab056ed4ec20a2e65af3cb5bcf62
-size 3944622
+oid sha256:88a27b7667a13cf00bf0643bf1cfabb2c186f4456de887228d9bc0e4331459fd
+size 3965521

submission_artifacts/planner.json

@@ -3,17 +3,21 @@
   "artifact": "/work/submission_artifacts/planner.joblib",
   "quantile": 0.05,
   "event_risk_threshold": 0.5,
-  "prediction_offset_days": -4.0,
-  "risk_calibration_scale": 1.85,
-  "expected_gain_margin": 2.5,
+  "prediction_offset_days": -5.0,
+  "risk_calibration_scale": 1.5,
+  "expected_gain_margin": 10.0,
   "expected_service_cost_hours": 2.0,
+  "minimum_scheduled_batteries": 16,
+  "maximum_scheduled_batteries": 20,
   "calibration_folds": 5,
   "planner_policy": {
     "expected_cost": true,
-    "risk_calibration_scale": 1.85,
+    "risk_calibration_scale": 1.5,
     "expected_service_cost_hours": 2.0,
-    "expected_gain_margin": 2.5,
-    "prediction_offset_days": -4.0,
+    "expected_gain_margin": 10.0,
+    "minimum_scheduled_batteries": 16,
+    "maximum_scheduled_batteries": 20,
+    "prediction_offset_days": -5.0,
     "stale_risk_cutoff_days": 7.0,
     "recent_gap_risk_factor": 0.75,
     "stale_risk_factor": 0.1,
@@ -97,16 +101,13 @@
     "voltage_range_day",
     "location_age_days",
     "data_gap_days",
-    "censor_proxy_rul_days",
-    "scenario_month_sin",
-    "scenario_month_cos",
     "voltage_rank_global",
     "voltage_rank_building",
     "voltage_rank_room",
     "building_battery_count",
     "room_battery_count"
   ],
-  "artifact_sha256": "75ed285b5e64b69543d58dc800bf50f508cdab056ed4ec20a2e65af3cb5bcf62",
+  "artifact_sha256": "88a27b7667a13cf00bf0643bf1cfabb2c186f4456de887228d9bc0e4331459fd",
   "runtime_versions": {
     "batteryswap_public": "0.3.4",
     "fastparquet": "2026.5.0",