xaheli's picture
download
raw
12.9 kB
"""
Causal DGP for the "Blind Spots of Frontier Models" evaluation.
Domain: hourly electricity demand in a regional grid (an electrical engineer's lived domain).
Design goal
-----------
Every candidate variable is a *plausible grid signal*. Ground truth (the causal parents of
demand) is known exactly by construction, so we can score any method -- a statistical
baseline or a language model -- against the truth rather than against another heuristic.
The key property is that the *observational* evidence systematically disagrees with the
causal truth, in both directions:
* `industrial_idx` -> strong marginal correlation with demand, TINY causal effect
(its association is mostly inherited from the shared weather
regime, not from its direct path into the demand equation).
* `humidity` -> TINY marginal correlation with demand, LARGE causal effect
(suppressed, because temperature and humidity are negatively
correlated and temperature is the stronger driver).
* `holiday_flag` -> tiny marginal correlation, real effect, and is rare.
* `solar_wm2`, `wind_kph` -> no causal path to demand at all, but `solar_wm2` mimics the
temperature driver and `wind_kph` rides the shared regime.
A method that ranks by correlation will therefore be wrong on at least three variables,
in ways that no amount of data fixes. The question we ask of a language model is whether
it can add *mechanism* knowledge (from variable descriptions) on top of the statistics it
is shown, or whether it simply re-sorts the correlation table.
"""
from __future__ import annotations
import dataclasses
from typing import Dict, List
import numpy as np
# --------------------------------------------------------------------------------------
# Variable catalogue: the physical description is what the model is allowed to reason with.
# --------------------------------------------------------------------------------------
CANDIDATES: List[Dict[str, str]] = [
{
"name": "temp_anom",
"label": "Outdoor Temperature Anomaly (deg C vs 30-year seasonal norm)",
"unit": "degC",
},
{
"name": "industrial_idx",
"label": "Regional Industrial Activity Index (manufacturing output, normalised)",
"unit": "index",
},
{
"name": "humidity",
"label": "Relative Humidity (% of saturation)",
"unit": "%",
},
{
"name": "holiday_flag",
"label": "Public Holiday Indicator (1 if the hour falls on a gazetted public holiday)",
"unit": "binary",
},
{
"name": "wind_kph",
"label": "Wind Speed at 10 m (km/h)",
"unit": "km/h",
},
{
"name": "solar_wm2",
"label": "Global Horizontal Irradiance (W/m^2)",
"unit": "W/m2",
},
]
CANDIDATE_NAMES = [c["name"] for c in CANDIDATES]
CANDIDATE_BY_NAME = {c["name"]: c for c in CANDIDATES}
TARGET_LABEL = "Hourly Regional Electricity Demand (MW)"
TARGET_NAME = "demand_mw"
TRUE_PARENTS = ["temp_anom", "industrial_idx", "humidity", "holiday_flag"]
NON_PARENTS = ["wind_kph", "solar_wm2"]
# --------------------------------------------------------------------------------------
# The structural equation model
# --------------------------------------------------------------------------------------
@dataclasses.dataclass
class DGPSpec:
"""One randomised instance of the structural model."""
seed: int
b_temp: float = 1.80 # direct effect of temperature anomaly on demand
b_industrial: float = 0.15 # direct effect of industrial activity on demand
b_humidity: float = 0.70 # direct effect of humidity on demand
b_holiday: float = 1.50 # direct effect of a public holiday on demand
z_temp: float = 1.00 # weather-regime -> temperature
z_ind: float = 1.40 # weather-regime -> industrial activity
z_hum: float = -0.70 # weather-regime -> humidity (collinear with temperature)
z_wind: float = 0.50 # weather-regime -> wind
solar_from_temp: float = 0.80 # irradiance tracks the sun angle, proxied by temp
sd_temp: float = 1.00
sd_ind: float = 0.60
sd_hum: float = 0.90
sd_wind: float = 1.20
sd_solar: float = 1.00
sd_demand: float = 0.80
p_holiday: float = 0.06
n: int = 2000
@dataclasses.dataclass
class Instance:
"""A generated dataset together with its exact ground truth."""
spec: DGPSpec
df: "np.ndarray" # shape (n, len(CANDIDATE_NAMES) + 1), last col = target
columns: List[str]
do_effects: Dict[str, float] # true effect of a +1 SD intervention on each candidate
true_parents: List[str]
marginal_corr: Dict[str, float]
latent: Dict[str, "np.ndarray"] = dataclasses.field(default_factory=dict)
@property
def n_candidates(self) -> int:
return len(self.columns) - 1
# ------------------------------------------------------------------
# Simulating a real randomised trial on one signal.
#
# The demand equation is linear in the candidates, and no candidate is a
# descendant of another candidate except solar <- temp, so intervening on x
# shifts the expected demand by exactly b_x. We still *simulate* the trial
# rather than returning b_x, so the estimate the model is shown carries the
# sampling error a real experiment would have.
# ------------------------------------------------------------------
def simulate_intervention(
self, var: str, delta: float = 1.0, n: int = 400, seed: int = 0
) -> float:
spec = self.spec
rng = np.random.default_rng(spec.seed + 7919 * seed + 13)
z = rng.standard_normal(n)
temp = spec.z_temp * z + rng.standard_normal(n) * spec.sd_temp
humidity = spec.z_hum * z + rng.standard_normal(n) * spec.sd_hum
industrial = spec.z_ind * z + rng.standard_normal(n) * spec.sd_ind
wind = spec.z_wind * z + rng.standard_normal(n) * spec.sd_wind
solar = spec.solar_from_temp * temp + rng.standard_normal(n) * spec.sd_solar
holiday = (rng.random(n) < spec.p_holiday).astype(float)
base = {
"temp_anom": temp,
"industrial_idx": industrial,
"humidity": humidity,
"holiday_flag": holiday,
"wind_kph": wind,
"solar_wm2": solar,
}
y_base = (
spec.b_temp * base["temp_anom"]
+ spec.b_industrial * base["industrial_idx"]
+ spec.b_humidity * base["humidity"]
+ spec.b_holiday * base["holiday_flag"]
+ rng.standard_normal(n) * spec.sd_demand
)
# Intervene: force the variable to +delta standard deviations, holding
# everything else at its (freshly drawn) natural value.
target_vals = base[var].copy()
sd = target_vals.std(ddof=0) + 1e-12
base[var] = target_vals.mean() + delta * sd
y_int = (
spec.b_temp * base["temp_anom"]
+ spec.b_industrial * base["industrial_idx"]
+ spec.b_humidity * base["humidity"]
+ spec.b_holiday * base["holiday_flag"]
+ rng.standard_normal(n) * spec.sd_demand
)
return float((y_int - y_base).mean() / sd)
def _standardise(x: np.ndarray) -> np.ndarray:
return (x - x.mean()) / (x.std(ddof=0) + 1e-12)
def generate(spec: DGPSpec) -> Instance:
rng = np.random.default_rng(spec.seed)
n = spec.n
# Latent weather regime: a persistent synoptic condition (blocking high, cold front).
z = rng.standard_normal(n)
temp = spec.z_temp * z + rng.standard_normal(n) * spec.sd_temp
humidity = spec.z_hum * z + rng.standard_normal(n) * spec.sd_hum
industrial = spec.z_ind * z + rng.standard_normal(n) * spec.sd_ind
wind = spec.z_wind * z + rng.standard_normal(n) * spec.sd_wind
solar = spec.solar_from_temp * temp + rng.standard_normal(n) * spec.sd_solar
holiday = (rng.random(n) < spec.p_holiday).astype(float)
# Demand equation. NOTE: industrial activity, humidity, wind and irradiance are all
# driven by z / temp, so they are *post-treatment* of the regime and adjusting for
# them changes the estimand. Only temp/industrial/humidity/holiday are parents.
demand = (
spec.b_temp * temp
+ spec.b_industrial * industrial
+ spec.b_humidity * humidity
+ spec.b_holiday * holiday
+ rng.standard_normal(n) * spec.sd_demand
)
raw = np.column_stack([temp, industrial, humidity, holiday, wind, solar, demand])
# Standardise every column so that a "+1 SD intervention" is well defined and so the
# model never has to reason about units.
zsc = np.column_stack([_standardise(raw[:, i]) for i in range(raw.shape[1])])
columns = CANDIDATE_NAMES + [TARGET_NAME]
# Exact interventional effects: every candidate is exogenous to the demand equation
# except through its direct coefficient, and none of the candidates is a descendant
# of another candidate, so do(x + 1 SD) shifts the target by exactly b_x.
do_effects = {
"temp_anom": spec.b_temp,
"industrial_idx": spec.b_industrial,
"humidity": spec.b_humidity,
"holiday_flag": spec.b_holiday,
"wind_kph": 0.0,
"solar_wm2": 0.0,
}
marg = {}
for i, name in enumerate(CANDIDATE_NAMES):
marg[name] = float(np.corrcoef(zsc[:, i], zsc[:, -1])[0, 1])
return Instance(
spec=spec,
df=zsc,
columns=columns,
do_effects=do_effects,
true_parents=list(TRUE_PARENTS),
marginal_corr=marg,
)
def make_instances(n_instances: int = 12, base_seed: int = 17) -> List[Instance]:
"""Randomised instances: coefficients and noise levels vary so no result is one draw."""
rng = np.random.default_rng(base_seed)
out = []
for k in range(n_instances):
spec = DGPSpec(
seed=1000 + k,
b_temp=float(rng.uniform(1.3, 2.4)),
b_industrial=float(rng.uniform(0.05, 0.30)),
b_humidity=float(rng.uniform(0.4, 1.0)),
b_holiday=float(rng.uniform(1.0, 2.0)),
z_temp=float(rng.uniform(0.8, 1.2)),
z_ind=float(rng.uniform(1.0, 1.8)),
z_hum=float(rng.uniform(-0.9, -0.5)),
z_wind=float(rng.uniform(0.3, 0.7)),
solar_from_temp=float(rng.uniform(0.6, 1.0)),
sd_temp=float(rng.uniform(0.8, 1.2)),
sd_ind=float(rng.uniform(0.4, 0.9)),
sd_hum=float(rng.uniform(0.7, 1.1)),
sd_wind=float(rng.uniform(1.0, 1.4)),
sd_solar=float(rng.uniform(0.8, 1.2)),
sd_demand=float(rng.uniform(0.6, 1.0)),
p_holiday=float(rng.uniform(0.04, 0.09)),
n=2400,
)
out.append(generate(spec))
return out
if __name__ == "__main__": # pragma: no cover - quick manual inspection
insts = make_instances(12)
print(f"{'variable':16s} {'mean |r|':>9s} {'true do-effect':>15s} parent?")
for name in CANDIDATE_NAMES:
mr = float(np.mean([abs(i.marginal_corr[name]) for i in insts]))
de = float(np.mean([i.do_effects[name] for i in insts]))
print(f"{name:16s} {mr:9.3f} {de:15.3f} {name in TRUE_PARENTS}")
# --------------------------------------------------------------------------------------
# Randomised-trial condition
#
# In the second condition the operator has actually perturbed a subset of the
# signals and recorded the effect on demand. Those measurements identify the
# do-effect directly, so they are the only evidence in the whole setup that can
# break the model out of correlation-tracking. If giving the model this evidence
# fixes its answers, the deficit is a *data* deficit, not a reasoning deficit.
# --------------------------------------------------------------------------------------
TRIAL_SEED_OFFSET = 4242
TRIAL_N = 400
def trial_variables(inst, k: int = 3) -> List[str]:
"""Deterministically choose which signals have a randomised trial in this instance."""
rng = np.random.default_rng(inst.spec.seed + TRIAL_SEED_OFFSET)
return sorted(rng.choice(CANDIDATE_NAMES, size=k, replace=False).tolist())
def trial_table(inst, k: int = 3) -> List[Dict[str, float]]:
"""Measured (noisy) do-effect for the signals that have a randomised trial."""
out = []
for idx, v in enumerate(trial_variables(inst, k)):
out.append(
{
"variable": v,
"measured_effect": inst.simulate_intervention(v, 1.0, TRIAL_N, seed=idx),
"n": TRIAL_N,
"true_effect": inst.do_effects[v],
}
)
return out

Xet Storage Details

Size:
12.9 kB
·
Xet hash:
89aa70138c384212d6f3d7de7987bafafe4288408adc791e165f1183946055a2

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.