Buckets:
| """ | |
| Causal DGP for the "Blind Spots of Frontier Models" evaluation. | |
| Domain: hourly electricity demand in a regional grid (an electrical engineer's lived domain). | |
| Design goal | |
| ----------- | |
| Every candidate variable is a *plausible grid signal*. Ground truth (the causal parents of | |
| demand) is known exactly by construction, so we can score any method -- a statistical | |
| baseline or a language model -- against the truth rather than against another heuristic. | |
| The key property is that the *observational* evidence systematically disagrees with the | |
| causal truth, in both directions: | |
| * `industrial_idx` -> strong marginal correlation with demand, TINY causal effect | |
| (its association is mostly inherited from the shared weather | |
| regime, not from its direct path into the demand equation). | |
| * `humidity` -> TINY marginal correlation with demand, LARGE causal effect | |
| (suppressed, because temperature and humidity are negatively | |
| correlated and temperature is the stronger driver). | |
| * `holiday_flag` -> tiny marginal correlation, real effect, and is rare. | |
| * `solar_wm2`, `wind_kph` -> no causal path to demand at all, but `solar_wm2` mimics the | |
| temperature driver and `wind_kph` rides the shared regime. | |
| A method that ranks by correlation will therefore be wrong on at least three variables, | |
| in ways that no amount of data fixes. The question we ask of a language model is whether | |
| it can add *mechanism* knowledge (from variable descriptions) on top of the statistics it | |
| is shown, or whether it simply re-sorts the correlation table. | |
| """ | |
| from __future__ import annotations | |
| import dataclasses | |
| from typing import Dict, List | |
| import numpy as np | |
| # -------------------------------------------------------------------------------------- | |
| # Variable catalogue: the physical description is what the model is allowed to reason with. | |
| # -------------------------------------------------------------------------------------- | |
| CANDIDATES: List[Dict[str, str]] = [ | |
| { | |
| "name": "temp_anom", | |
| "label": "Outdoor Temperature Anomaly (deg C vs 30-year seasonal norm)", | |
| "unit": "degC", | |
| }, | |
| { | |
| "name": "industrial_idx", | |
| "label": "Regional Industrial Activity Index (manufacturing output, normalised)", | |
| "unit": "index", | |
| }, | |
| { | |
| "name": "humidity", | |
| "label": "Relative Humidity (% of saturation)", | |
| "unit": "%", | |
| }, | |
| { | |
| "name": "holiday_flag", | |
| "label": "Public Holiday Indicator (1 if the hour falls on a gazetted public holiday)", | |
| "unit": "binary", | |
| }, | |
| { | |
| "name": "wind_kph", | |
| "label": "Wind Speed at 10 m (km/h)", | |
| "unit": "km/h", | |
| }, | |
| { | |
| "name": "solar_wm2", | |
| "label": "Global Horizontal Irradiance (W/m^2)", | |
| "unit": "W/m2", | |
| }, | |
| ] | |
| CANDIDATE_NAMES = [c["name"] for c in CANDIDATES] | |
| CANDIDATE_BY_NAME = {c["name"]: c for c in CANDIDATES} | |
| TARGET_LABEL = "Hourly Regional Electricity Demand (MW)" | |
| TARGET_NAME = "demand_mw" | |
| TRUE_PARENTS = ["temp_anom", "industrial_idx", "humidity", "holiday_flag"] | |
| NON_PARENTS = ["wind_kph", "solar_wm2"] | |
| # -------------------------------------------------------------------------------------- | |
| # The structural equation model | |
| # -------------------------------------------------------------------------------------- | |
| class DGPSpec: | |
| """One randomised instance of the structural model.""" | |
| seed: int | |
| b_temp: float = 1.80 # direct effect of temperature anomaly on demand | |
| b_industrial: float = 0.15 # direct effect of industrial activity on demand | |
| b_humidity: float = 0.70 # direct effect of humidity on demand | |
| b_holiday: float = 1.50 # direct effect of a public holiday on demand | |
| z_temp: float = 1.00 # weather-regime -> temperature | |
| z_ind: float = 1.40 # weather-regime -> industrial activity | |
| z_hum: float = -0.70 # weather-regime -> humidity (collinear with temperature) | |
| z_wind: float = 0.50 # weather-regime -> wind | |
| solar_from_temp: float = 0.80 # irradiance tracks the sun angle, proxied by temp | |
| sd_temp: float = 1.00 | |
| sd_ind: float = 0.60 | |
| sd_hum: float = 0.90 | |
| sd_wind: float = 1.20 | |
| sd_solar: float = 1.00 | |
| sd_demand: float = 0.80 | |
| p_holiday: float = 0.06 | |
| n: int = 2000 | |
| class Instance: | |
| """A generated dataset together with its exact ground truth.""" | |
| spec: DGPSpec | |
| df: "np.ndarray" # shape (n, len(CANDIDATE_NAMES) + 1), last col = target | |
| columns: List[str] | |
| do_effects: Dict[str, float] # true effect of a +1 SD intervention on each candidate | |
| true_parents: List[str] | |
| marginal_corr: Dict[str, float] | |
| latent: Dict[str, "np.ndarray"] = dataclasses.field(default_factory=dict) | |
| def n_candidates(self) -> int: | |
| return len(self.columns) - 1 | |
| # ------------------------------------------------------------------ | |
| # Simulating a real randomised trial on one signal. | |
| # | |
| # The demand equation is linear in the candidates, and no candidate is a | |
| # descendant of another candidate except solar <- temp, so intervening on x | |
| # shifts the expected demand by exactly b_x. We still *simulate* the trial | |
| # rather than returning b_x, so the estimate the model is shown carries the | |
| # sampling error a real experiment would have. | |
| # ------------------------------------------------------------------ | |
| def simulate_intervention( | |
| self, var: str, delta: float = 1.0, n: int = 400, seed: int = 0 | |
| ) -> float: | |
| spec = self.spec | |
| rng = np.random.default_rng(spec.seed + 7919 * seed + 13) | |
| z = rng.standard_normal(n) | |
| temp = spec.z_temp * z + rng.standard_normal(n) * spec.sd_temp | |
| humidity = spec.z_hum * z + rng.standard_normal(n) * spec.sd_hum | |
| industrial = spec.z_ind * z + rng.standard_normal(n) * spec.sd_ind | |
| wind = spec.z_wind * z + rng.standard_normal(n) * spec.sd_wind | |
| solar = spec.solar_from_temp * temp + rng.standard_normal(n) * spec.sd_solar | |
| holiday = (rng.random(n) < spec.p_holiday).astype(float) | |
| base = { | |
| "temp_anom": temp, | |
| "industrial_idx": industrial, | |
| "humidity": humidity, | |
| "holiday_flag": holiday, | |
| "wind_kph": wind, | |
| "solar_wm2": solar, | |
| } | |
| y_base = ( | |
| spec.b_temp * base["temp_anom"] | |
| + spec.b_industrial * base["industrial_idx"] | |
| + spec.b_humidity * base["humidity"] | |
| + spec.b_holiday * base["holiday_flag"] | |
| + rng.standard_normal(n) * spec.sd_demand | |
| ) | |
| # Intervene: force the variable to +delta standard deviations, holding | |
| # everything else at its (freshly drawn) natural value. | |
| target_vals = base[var].copy() | |
| sd = target_vals.std(ddof=0) + 1e-12 | |
| base[var] = target_vals.mean() + delta * sd | |
| y_int = ( | |
| spec.b_temp * base["temp_anom"] | |
| + spec.b_industrial * base["industrial_idx"] | |
| + spec.b_humidity * base["humidity"] | |
| + spec.b_holiday * base["holiday_flag"] | |
| + rng.standard_normal(n) * spec.sd_demand | |
| ) | |
| return float((y_int - y_base).mean() / sd) | |
| def _standardise(x: np.ndarray) -> np.ndarray: | |
| return (x - x.mean()) / (x.std(ddof=0) + 1e-12) | |
| def generate(spec: DGPSpec) -> Instance: | |
| rng = np.random.default_rng(spec.seed) | |
| n = spec.n | |
| # Latent weather regime: a persistent synoptic condition (blocking high, cold front). | |
| z = rng.standard_normal(n) | |
| temp = spec.z_temp * z + rng.standard_normal(n) * spec.sd_temp | |
| humidity = spec.z_hum * z + rng.standard_normal(n) * spec.sd_hum | |
| industrial = spec.z_ind * z + rng.standard_normal(n) * spec.sd_ind | |
| wind = spec.z_wind * z + rng.standard_normal(n) * spec.sd_wind | |
| solar = spec.solar_from_temp * temp + rng.standard_normal(n) * spec.sd_solar | |
| holiday = (rng.random(n) < spec.p_holiday).astype(float) | |
| # Demand equation. NOTE: industrial activity, humidity, wind and irradiance are all | |
| # driven by z / temp, so they are *post-treatment* of the regime and adjusting for | |
| # them changes the estimand. Only temp/industrial/humidity/holiday are parents. | |
| demand = ( | |
| spec.b_temp * temp | |
| + spec.b_industrial * industrial | |
| + spec.b_humidity * humidity | |
| + spec.b_holiday * holiday | |
| + rng.standard_normal(n) * spec.sd_demand | |
| ) | |
| raw = np.column_stack([temp, industrial, humidity, holiday, wind, solar, demand]) | |
| # Standardise every column so that a "+1 SD intervention" is well defined and so the | |
| # model never has to reason about units. | |
| zsc = np.column_stack([_standardise(raw[:, i]) for i in range(raw.shape[1])]) | |
| columns = CANDIDATE_NAMES + [TARGET_NAME] | |
| # Exact interventional effects: every candidate is exogenous to the demand equation | |
| # except through its direct coefficient, and none of the candidates is a descendant | |
| # of another candidate, so do(x + 1 SD) shifts the target by exactly b_x. | |
| do_effects = { | |
| "temp_anom": spec.b_temp, | |
| "industrial_idx": spec.b_industrial, | |
| "humidity": spec.b_humidity, | |
| "holiday_flag": spec.b_holiday, | |
| "wind_kph": 0.0, | |
| "solar_wm2": 0.0, | |
| } | |
| marg = {} | |
| for i, name in enumerate(CANDIDATE_NAMES): | |
| marg[name] = float(np.corrcoef(zsc[:, i], zsc[:, -1])[0, 1]) | |
| return Instance( | |
| spec=spec, | |
| df=zsc, | |
| columns=columns, | |
| do_effects=do_effects, | |
| true_parents=list(TRUE_PARENTS), | |
| marginal_corr=marg, | |
| ) | |
| def make_instances(n_instances: int = 12, base_seed: int = 17) -> List[Instance]: | |
| """Randomised instances: coefficients and noise levels vary so no result is one draw.""" | |
| rng = np.random.default_rng(base_seed) | |
| out = [] | |
| for k in range(n_instances): | |
| spec = DGPSpec( | |
| seed=1000 + k, | |
| b_temp=float(rng.uniform(1.3, 2.4)), | |
| b_industrial=float(rng.uniform(0.05, 0.30)), | |
| b_humidity=float(rng.uniform(0.4, 1.0)), | |
| b_holiday=float(rng.uniform(1.0, 2.0)), | |
| z_temp=float(rng.uniform(0.8, 1.2)), | |
| z_ind=float(rng.uniform(1.0, 1.8)), | |
| z_hum=float(rng.uniform(-0.9, -0.5)), | |
| z_wind=float(rng.uniform(0.3, 0.7)), | |
| solar_from_temp=float(rng.uniform(0.6, 1.0)), | |
| sd_temp=float(rng.uniform(0.8, 1.2)), | |
| sd_ind=float(rng.uniform(0.4, 0.9)), | |
| sd_hum=float(rng.uniform(0.7, 1.1)), | |
| sd_wind=float(rng.uniform(1.0, 1.4)), | |
| sd_solar=float(rng.uniform(0.8, 1.2)), | |
| sd_demand=float(rng.uniform(0.6, 1.0)), | |
| p_holiday=float(rng.uniform(0.04, 0.09)), | |
| n=2400, | |
| ) | |
| out.append(generate(spec)) | |
| return out | |
| if __name__ == "__main__": # pragma: no cover - quick manual inspection | |
| insts = make_instances(12) | |
| print(f"{'variable':16s} {'mean |r|':>9s} {'true do-effect':>15s} parent?") | |
| for name in CANDIDATE_NAMES: | |
| mr = float(np.mean([abs(i.marginal_corr[name]) for i in insts])) | |
| de = float(np.mean([i.do_effects[name] for i in insts])) | |
| print(f"{name:16s} {mr:9.3f} {de:15.3f} {name in TRUE_PARENTS}") | |
| # -------------------------------------------------------------------------------------- | |
| # Randomised-trial condition | |
| # | |
| # In the second condition the operator has actually perturbed a subset of the | |
| # signals and recorded the effect on demand. Those measurements identify the | |
| # do-effect directly, so they are the only evidence in the whole setup that can | |
| # break the model out of correlation-tracking. If giving the model this evidence | |
| # fixes its answers, the deficit is a *data* deficit, not a reasoning deficit. | |
| # -------------------------------------------------------------------------------------- | |
| TRIAL_SEED_OFFSET = 4242 | |
| TRIAL_N = 400 | |
| def trial_variables(inst, k: int = 3) -> List[str]: | |
| """Deterministically choose which signals have a randomised trial in this instance.""" | |
| rng = np.random.default_rng(inst.spec.seed + TRIAL_SEED_OFFSET) | |
| return sorted(rng.choice(CANDIDATE_NAMES, size=k, replace=False).tolist()) | |
| def trial_table(inst, k: int = 3) -> List[Dict[str, float]]: | |
| """Measured (noisy) do-effect for the signals that have a randomised trial.""" | |
| out = [] | |
| for idx, v in enumerate(trial_variables(inst, k)): | |
| out.append( | |
| { | |
| "variable": v, | |
| "measured_effect": inst.simulate_intervention(v, 1.0, TRIAL_N, seed=idx), | |
| "n": TRIAL_N, | |
| "true_effect": inst.do_effects[v], | |
| } | |
| ) | |
| return out | |
Xet Storage Details
- Size:
- 12.9 kB
- Xet hash:
- 89aa70138c384212d6f3d7de7987bafafe4288408adc791e165f1183946055a2
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.