xaheli's picture
download
raw
16.5 kB
"""
Blind Spots of Frontier Models -- evaluation harness.
Two conditions, five tasks, all on the same synthetic-but-ground-truthed grid-demand
instances.
OBSERVATIONAL CONDITION (the model's default setting)
select which signals DIRECTLY CAUSE the target?
predict which signals best PREDICT the target?
intervene for a given signal, what is the effect of intervening on it by +1 SD?
RANDOMISED-TRIAL CONDITION (the model additionally receives real interventional data)
select_trial same causal question, with trial evidence for 3 of the 6 signals
intervene_trial same intervention question, with trial evidence for 3 of the 6 signals
The model is shown: variable names, physical descriptions, a small row sample, and the
marginal correlation of each candidate with the target. It is NOT shown the causal
graph. Everything it says about mechanism has to come from its own knowledge -- or,
in the trial condition, from the interventional measurements it is given.
"""
from __future__ import annotations
import argparse
import json
import os
import re
import time
from typing import Dict, List
import numpy as np
import torch
from dgp import (
CANDIDATE_BY_NAME,
CANDIDATE_NAMES,
TARGET_LABEL,
TARGET_NAME,
make_instances,
trial_table,
trial_variables,
)
# --------------------------------------------------------------------------------------
# Model zoo. All open-weight and inside the 0.6B-6B window required by the brief.
# --------------------------------------------------------------------------------------
MODELS: List[Dict[str, object]] = [
{"id": "Qwen/Qwen3-4B", "label": "Qwen3-4B", "thinking": False},
{"id": "Qwen/Qwen3-1.7B", "label": "Qwen3-1.7B", "thinking": False},
{"id": "Qwen/Qwen3-0.6B", "label": "Qwen3-0.6B", "thinking": False},
{"id": "microsoft/Phi-4-mini-instruct", "label": "Phi-4-mini-instruct", "thinking": False},
{"id": "HuggingFaceTB/SmolLM3-3B", "label": "SmolLM3-3B", "thinking": False},
]
ALL_TASKS = ("select", "predict", "intervene", "select_trial", "intervene_trial")
N_SAMPLE_ROWS = 8
# --------------------------------------------------------------------------------------
# Prompt construction
# --------------------------------------------------------------------------------------
def _candidate_block() -> str:
return "\n".join(
f" - {c['name']}: {c['label']}" for c in (CANDIDATE_BY_NAME[n] for n in CANDIDATE_NAMES)
)
def _corr_block(inst) -> str:
return "\n".join(f" - {n}: r = {inst.marginal_corr[n]:+.2f}" for n in CANDIDATE_NAMES)
def _sample_table(inst) -> str:
idx = np.linspace(0, inst.df.shape[0] - 1, N_SAMPLE_ROWS).astype(int)
rows = inst.df[idx]
header = " | " + " | ".join(CANDIDATE_NAMES + [TARGET_NAME]) + " |"
sep = " |" + "|".join(["---"] * (len(CANDIDATE_NAMES) + 1)) + "|"
body = []
for r in rows:
body.append(" | " + " | ".join(f"{v:+.2f}" for v in r) + " |")
return "\n".join([header, sep] + body)
def _preamble(inst) -> str:
return f"""You are a grid systems analyst and research scientist. You understand power systems, statistics and causal inference at a research level.
A regional grid operator recorded hourly electricity demand together with a set of candidate signals, over N = {inst.spec.n} hours. All values below are standardised (mean 0, sd 1).
TARGET: {TARGET_LABEL}
CANDIDATE SIGNALS:
{_candidate_block()}
MARGINAL PEARSON CORRELATION OF EACH SIGNAL WITH THE TARGET (computed on the observational data):
{_corr_block(inst)}
A RANDOM SAMPLE OF {N_SAMPLE_ROWS} HOURS (columns are standardised values):
{_sample_table(inst)}
"""
def _trial_block(inst) -> str:
return "\n".join(
" - {var}: measured change in demand = {eff:+.2f} sd (n={n} randomised)".format(
var=t["variable"], eff=t["measured_effect"], n=t["n"]
)
for t in trial_table(inst)
)
def _trial_preamble(inst) -> str:
return (
_preamble(inst)
+ f"""
RANDOMISED TRIAL EVIDENCE (new information):
The operator was able to run a randomised trial on some of the signals: for each signal below, the signal was deliberately shifted upward by one standard deviation while all other signals were held at their natural values, and the resulting change in demand was measured over 400 independent repetitions.
{_trial_block(inst)}
The signals NOT listed above were never experimentally perturbed, so for those you have only the observational evidence above.
"""
)
def prompt_select(inst) -> str:
return _preamble(inst) + """
TASK: Which of the candidate signals are DIRECT CAUSES of hourly electricity demand?
A cause is a signal such that intervening on it directly -- changing it while holding everything else fixed -- would change the expected demand. Use the physical mechanism, not the correlation value. A signal that is only correlated with demand, or that is merely a proxy for some other driver, is NOT a cause of it. A weak correlation does not rule out a real mechanism, and a strong correlation does not establish one.
Consider each candidate in turn and reason about the physical pathway.
Return ONLY a JSON object of exactly this form, with no text before or after it:
{"causes": ["<signal name>", "<signal name>"], "reasoning": "<two or three sentences explaining the mechanism for each signal you selected>"}
Use the exact signal names given above.
"""
def prompt_predict(inst) -> str:
return _preamble(inst) + """
TASK: Which of the candidate signals are most PREDICTIVE of hourly electricity demand? Use the statistical evidence: correlation, the sample, and any nonlinear structure you can infer.
Return ONLY a JSON object of exactly this form, with no text before or after it:
{"predictors": ["<signal name>", "<signal name>"], "reasoning": "<two or three sentences>"}
Use the exact signal names given above.
"""
def prompt_intervene(inst, var: str) -> str:
return _preamble(inst) + f"""
TASK: Consider the candidate signal `{var}`. Suppose an operator INTERVENES on it and shifts it upward by one standard deviation, holding all other signals at their observed values. By how much does expected hourly demand change, in units of the target's standard deviation?
Give a signed number. Think about the physical pathway from `{var}` to demand, and about which of the observed associations are genuine mechanisms versus inherited correlations from a shared cause.
Return ONLY a JSON object of exactly this form, with no text before or after it:
{{"effect": <signed number>, "reasoning": "<two or three sentences>"}}
"""
def prompt_select_trial(inst) -> str:
return _trial_preamble(inst) + """
TASK: Which of the candidate signals are DIRECT CAUSES of hourly electricity demand?
A cause is a signal such that intervening on it directly -- changing it while holding everything else fixed -- would change the expected demand. Use every piece of evidence you have: the randomised trial measurements are direct interventions and are decisive; the correlations are not, because a correlation can be inherited from a shared cause rather than transmitted through a mechanism. A weak correlation does not rule out a real mechanism, and a strong correlation does not establish one.
Consider each candidate in turn and reason about the physical pathway.
Return ONLY a JSON object of exactly this form, with no text before or after it:
{"causes": ["<signal name>", "<signal name>"], "reasoning": "<two or three sentences explaining the mechanism for each signal you selected>"}
Use the exact signal names given above.
"""
def prompt_intervene_trial(inst, var: str) -> str:
has_trial = var in trial_variables(inst)
extra = (
"\nNOTE: this signal has a randomised trial measurement above. Use it."
if has_trial
else "\nNOTE: this signal has NO randomised trial measurement, so you must infer its effect from the observational evidence and physical reasoning."
)
return _trial_preamble(inst) + f"""
TASK: Consider the candidate signal `{var}`. Suppose an operator INTERVENES on it and shifts it upward by one standard deviation, holding all other signals at their observed values. By how much does expected hourly demand change, in units of the target's standard deviation?{extra}
Give a signed number. Think about the physical pathway from `{var}` to demand, and about which of the observed associations are genuine mechanisms versus inherited correlations from a shared cause.
Return ONLY a JSON object of exactly this form, with no text before or after it:
{{"effect": <signed number>, "reasoning": "<two or three sentences>"}}
"""
# --------------------------------------------------------------------------------------
# Model plumbing
# --------------------------------------------------------------------------------------
def load_model(repo_id: str):
from transformers import AutoModelForCausalLM, AutoTokenizer
tok = AutoTokenizer.from_pretrained(repo_id)
tok.padding_side = "left"
if tok.pad_token is None:
tok.pad_token = tok.eos_token
model = AutoModelForCausalLM.from_pretrained(
repo_id, dtype=torch.bfloat16, device_map="cuda"
)
model.eval()
return tok, model
def _chat(tok, user_msg: str, thinking: bool) -> str:
msgs = [{"role": "user", "content": user_msg}]
try:
return tok.apply_chat_template(
msgs, tokenize=False, add_generation_prompt=True, enable_thinking=thinking
)
except TypeError:
return tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True)
@torch.no_grad()
def generate(tok, model, prompts: List[str], max_new_tokens: int = 512, batch_size: int = 8) -> List[str]:
outs: List[str] = []
for i in range(0, len(prompts), batch_size):
chunk = prompts[i : i + batch_size]
enc = tok(chunk, return_tensors="pt", padding=True, truncation=True, max_length=4096).to(
model.device
)
gen = model.generate(
**enc,
max_new_tokens=max_new_tokens,
do_sample=False,
temperature=None,
top_p=None,
top_k=None,
pad_token_id=tok.pad_token_id,
)
new = gen[:, enc["input_ids"].shape[1] :]
outs.extend(tok.batch_decode(new, skip_special_tokens=True))
print(f" ...{min(i + batch_size, len(prompts))}/{len(prompts)}", flush=True)
return outs
# --------------------------------------------------------------------------------------
# Parsing
# --------------------------------------------------------------------------------------
def _first_json(text: str):
m = re.search(r"\{.*?\}", text, re.S)
if not m:
return None
try:
return json.loads(m.group(0))
except Exception:
try:
return json.loads(re.sub(r",\s*([}\]])", r"\1", m.group(0)))
except Exception:
return None
def parse_names(text: str, key: str, allowed: List[str]) -> List[str]:
obj = _first_json(text)
names: List[str] = []
if isinstance(obj, dict) and isinstance(obj.get(key), list):
for v in obj[key]:
if isinstance(v, str):
v = v.strip().strip("`").lower().replace(" ", "_")
if v in allowed and v not in names:
names.append(v)
if not names: # fallback: scan for known names in the raw text
low = text.lower()
for n in allowed:
if n in low and n not in names:
names.append(n)
return names
def parse_effect(text: str):
obj = _first_json(text)
if isinstance(obj, dict) and "effect" in obj:
try:
return float(obj["effect"]), str(obj.get("reasoning", ""))
except Exception:
pass
m = re.search(r'"effect"\s*:\s*(-?\d+(?:\.\d+)?)', text)
if m:
return float(m.group(1)), ""
m = re.search(r"(-?\d+(?:\.\d+)?)", text)
if m:
try:
return float(m.group(1)), ""
except Exception:
return None, ""
return None, ""
# --------------------------------------------------------------------------------------
# Main
# --------------------------------------------------------------------------------------
def run_model(model_cfg, instances, out_dir: str, tasks=ALL_TASKS, merge: bool = False):
repo_id = str(model_cfg["id"])
label = str(model_cfg["label"])
thinking = bool(model_cfg["thinking"])
print(f"\n=== {label} ({repo_id}) ===", flush=True)
tok, model = load_model(repo_id)
results: Dict[str, List[dict]] = {t: [] for t in ALL_TASKS}
os.makedirs(out_dir, exist_ok=True)
path = os.path.join(out_dir, f"raw_{label.replace('/', '_')}.json")
if merge and os.path.exists(path):
prev = json.load(open(path, encoding="utf8"))
for k, v in prev.get("results", {}).items():
results.setdefault(k, [])
results[k] = v
if "select" in tasks:
prompts = [_chat(tok, prompt_select(i), thinking) for i in instances]
t0 = time.time()
outs = generate(tok, model, prompts)
print(f" select done in {time.time()-t0:.0f}s", flush=True)
for inst, out in zip(instances, outs):
results["select"].append(
{"instance": inst.spec.seed, "raw": out, "pred": parse_names(out, "causes", CANDIDATE_NAMES)}
)
if "predict" in tasks:
prompts = [_chat(tok, prompt_predict(i), thinking) for i in instances]
outs = generate(tok, model, prompts)
for inst, out in zip(instances, outs):
results["predict"].append(
{"instance": inst.spec.seed, "raw": out, "pred": parse_names(out, "predictors", CANDIDATE_NAMES)}
)
if "intervene" in tasks:
prompts, keys = [], []
for inst in instances:
for v in CANDIDATE_NAMES:
prompts.append(_chat(tok, prompt_intervene(inst, v), thinking))
keys.append((inst.spec.seed, v))
outs = generate(tok, model, prompts)
for (seed, var), out in zip(keys, outs):
eff, _ = parse_effect(out)
results["intervene"].append(
{"instance": seed, "variable": var, "effect": eff, "raw": out}
)
if "select_trial" in tasks:
prompts = [_chat(tok, prompt_select_trial(i), thinking) for i in instances]
outs = generate(tok, model, prompts)
for inst, out in zip(instances, outs):
results["select_trial"].append(
{"instance": inst.spec.seed, "raw": out, "pred": parse_names(out, "causes", CANDIDATE_NAMES)}
)
if "intervene_trial" in tasks:
prompts, keys = [], []
for inst in instances:
for v in CANDIDATE_NAMES:
prompts.append(_chat(tok, prompt_intervene_trial(inst, v), thinking))
keys.append((inst.spec.seed, v))
outs = generate(tok, model, prompts)
for (seed, var), out in zip(keys, outs):
eff, _ = parse_effect(out)
results["intervene_trial"].append(
{"instance": seed, "variable": var, "effect": eff, "raw": out}
)
with open(path, "w", encoding="utf8") as f:
json.dump({"model": repo_id, "label": label, "results": results}, f, indent=1)
print(f" wrote {path}", flush=True)
del model
torch.cuda.empty_cache()
return path
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--models", nargs="*", default=None, help="subset of model labels")
ap.add_argument("--instances", type=int, default=12)
ap.add_argument("--out", default="results")
ap.add_argument("--tasks", nargs="*", default=list(ALL_TASKS))
ap.add_argument("--merge", action="store_true")
args = ap.parse_args()
instances = make_instances(args.instances)
print(f"generated {len(instances)} instances; tasks={args.tasks}", flush=True)
os.makedirs(args.out, exist_ok=True)
for cfg in MODELS:
if args.models and cfg["label"] not in args.models:
continue
try:
run_model(cfg, instances, args.out, tasks=tuple(args.tasks), merge=args.merge)
except Exception as e:
print(f"!! {cfg['label']} failed: {type(e).__name__}: {e}", flush=True)
import traceback
traceback.print_exc()
if __name__ == "__main__":
main()

Xet Storage Details

Size:
16.5 kB
·
Xet hash:
ca61eefbdf5a1b81bcfa0293de1a026809193b4dbff91d575da2f4bb861ffa17

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.