Buckets:
| """ | |
| Blind Spots of Frontier Models -- evaluation harness. | |
| Two conditions, five tasks, all on the same synthetic-but-ground-truthed grid-demand | |
| instances. | |
| OBSERVATIONAL CONDITION (the model's default setting) | |
| select which signals DIRECTLY CAUSE the target? | |
| predict which signals best PREDICT the target? | |
| intervene for a given signal, what is the effect of intervening on it by +1 SD? | |
| RANDOMISED-TRIAL CONDITION (the model additionally receives real interventional data) | |
| select_trial same causal question, with trial evidence for 3 of the 6 signals | |
| intervene_trial same intervention question, with trial evidence for 3 of the 6 signals | |
| The model is shown: variable names, physical descriptions, a small row sample, and the | |
| marginal correlation of each candidate with the target. It is NOT shown the causal | |
| graph. Everything it says about mechanism has to come from its own knowledge -- or, | |
| in the trial condition, from the interventional measurements it is given. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import re | |
| import time | |
| from typing import Dict, List | |
| import numpy as np | |
| import torch | |
| from dgp import ( | |
| CANDIDATE_BY_NAME, | |
| CANDIDATE_NAMES, | |
| TARGET_LABEL, | |
| TARGET_NAME, | |
| make_instances, | |
| trial_table, | |
| trial_variables, | |
| ) | |
| # -------------------------------------------------------------------------------------- | |
| # Model zoo. All open-weight and inside the 0.6B-6B window required by the brief. | |
| # -------------------------------------------------------------------------------------- | |
| MODELS: List[Dict[str, object]] = [ | |
| {"id": "Qwen/Qwen3-4B", "label": "Qwen3-4B", "thinking": False}, | |
| {"id": "Qwen/Qwen3-1.7B", "label": "Qwen3-1.7B", "thinking": False}, | |
| {"id": "Qwen/Qwen3-0.6B", "label": "Qwen3-0.6B", "thinking": False}, | |
| {"id": "microsoft/Phi-4-mini-instruct", "label": "Phi-4-mini-instruct", "thinking": False}, | |
| {"id": "HuggingFaceTB/SmolLM3-3B", "label": "SmolLM3-3B", "thinking": False}, | |
| ] | |
| ALL_TASKS = ("select", "predict", "intervene", "select_trial", "intervene_trial") | |
| N_SAMPLE_ROWS = 8 | |
| # -------------------------------------------------------------------------------------- | |
| # Prompt construction | |
| # -------------------------------------------------------------------------------------- | |
| def _candidate_block() -> str: | |
| return "\n".join( | |
| f" - {c['name']}: {c['label']}" for c in (CANDIDATE_BY_NAME[n] for n in CANDIDATE_NAMES) | |
| ) | |
| def _corr_block(inst) -> str: | |
| return "\n".join(f" - {n}: r = {inst.marginal_corr[n]:+.2f}" for n in CANDIDATE_NAMES) | |
| def _sample_table(inst) -> str: | |
| idx = np.linspace(0, inst.df.shape[0] - 1, N_SAMPLE_ROWS).astype(int) | |
| rows = inst.df[idx] | |
| header = " | " + " | ".join(CANDIDATE_NAMES + [TARGET_NAME]) + " |" | |
| sep = " |" + "|".join(["---"] * (len(CANDIDATE_NAMES) + 1)) + "|" | |
| body = [] | |
| for r in rows: | |
| body.append(" | " + " | ".join(f"{v:+.2f}" for v in r) + " |") | |
| return "\n".join([header, sep] + body) | |
| def _preamble(inst) -> str: | |
| return f"""You are a grid systems analyst and research scientist. You understand power systems, statistics and causal inference at a research level. | |
| A regional grid operator recorded hourly electricity demand together with a set of candidate signals, over N = {inst.spec.n} hours. All values below are standardised (mean 0, sd 1). | |
| TARGET: {TARGET_LABEL} | |
| CANDIDATE SIGNALS: | |
| {_candidate_block()} | |
| MARGINAL PEARSON CORRELATION OF EACH SIGNAL WITH THE TARGET (computed on the observational data): | |
| {_corr_block(inst)} | |
| A RANDOM SAMPLE OF {N_SAMPLE_ROWS} HOURS (columns are standardised values): | |
| {_sample_table(inst)} | |
| """ | |
| def _trial_block(inst) -> str: | |
| return "\n".join( | |
| " - {var}: measured change in demand = {eff:+.2f} sd (n={n} randomised)".format( | |
| var=t["variable"], eff=t["measured_effect"], n=t["n"] | |
| ) | |
| for t in trial_table(inst) | |
| ) | |
| def _trial_preamble(inst) -> str: | |
| return ( | |
| _preamble(inst) | |
| + f""" | |
| RANDOMISED TRIAL EVIDENCE (new information): | |
| The operator was able to run a randomised trial on some of the signals: for each signal below, the signal was deliberately shifted upward by one standard deviation while all other signals were held at their natural values, and the resulting change in demand was measured over 400 independent repetitions. | |
| {_trial_block(inst)} | |
| The signals NOT listed above were never experimentally perturbed, so for those you have only the observational evidence above. | |
| """ | |
| ) | |
| def prompt_select(inst) -> str: | |
| return _preamble(inst) + """ | |
| TASK: Which of the candidate signals are DIRECT CAUSES of hourly electricity demand? | |
| A cause is a signal such that intervening on it directly -- changing it while holding everything else fixed -- would change the expected demand. Use the physical mechanism, not the correlation value. A signal that is only correlated with demand, or that is merely a proxy for some other driver, is NOT a cause of it. A weak correlation does not rule out a real mechanism, and a strong correlation does not establish one. | |
| Consider each candidate in turn and reason about the physical pathway. | |
| Return ONLY a JSON object of exactly this form, with no text before or after it: | |
| {"causes": ["<signal name>", "<signal name>"], "reasoning": "<two or three sentences explaining the mechanism for each signal you selected>"} | |
| Use the exact signal names given above. | |
| """ | |
| def prompt_predict(inst) -> str: | |
| return _preamble(inst) + """ | |
| TASK: Which of the candidate signals are most PREDICTIVE of hourly electricity demand? Use the statistical evidence: correlation, the sample, and any nonlinear structure you can infer. | |
| Return ONLY a JSON object of exactly this form, with no text before or after it: | |
| {"predictors": ["<signal name>", "<signal name>"], "reasoning": "<two or three sentences>"} | |
| Use the exact signal names given above. | |
| """ | |
| def prompt_intervene(inst, var: str) -> str: | |
| return _preamble(inst) + f""" | |
| TASK: Consider the candidate signal `{var}`. Suppose an operator INTERVENES on it and shifts it upward by one standard deviation, holding all other signals at their observed values. By how much does expected hourly demand change, in units of the target's standard deviation? | |
| Give a signed number. Think about the physical pathway from `{var}` to demand, and about which of the observed associations are genuine mechanisms versus inherited correlations from a shared cause. | |
| Return ONLY a JSON object of exactly this form, with no text before or after it: | |
| {{"effect": <signed number>, "reasoning": "<two or three sentences>"}} | |
| """ | |
| def prompt_select_trial(inst) -> str: | |
| return _trial_preamble(inst) + """ | |
| TASK: Which of the candidate signals are DIRECT CAUSES of hourly electricity demand? | |
| A cause is a signal such that intervening on it directly -- changing it while holding everything else fixed -- would change the expected demand. Use every piece of evidence you have: the randomised trial measurements are direct interventions and are decisive; the correlations are not, because a correlation can be inherited from a shared cause rather than transmitted through a mechanism. A weak correlation does not rule out a real mechanism, and a strong correlation does not establish one. | |
| Consider each candidate in turn and reason about the physical pathway. | |
| Return ONLY a JSON object of exactly this form, with no text before or after it: | |
| {"causes": ["<signal name>", "<signal name>"], "reasoning": "<two or three sentences explaining the mechanism for each signal you selected>"} | |
| Use the exact signal names given above. | |
| """ | |
| def prompt_intervene_trial(inst, var: str) -> str: | |
| has_trial = var in trial_variables(inst) | |
| extra = ( | |
| "\nNOTE: this signal has a randomised trial measurement above. Use it." | |
| if has_trial | |
| else "\nNOTE: this signal has NO randomised trial measurement, so you must infer its effect from the observational evidence and physical reasoning." | |
| ) | |
| return _trial_preamble(inst) + f""" | |
| TASK: Consider the candidate signal `{var}`. Suppose an operator INTERVENES on it and shifts it upward by one standard deviation, holding all other signals at their observed values. By how much does expected hourly demand change, in units of the target's standard deviation?{extra} | |
| Give a signed number. Think about the physical pathway from `{var}` to demand, and about which of the observed associations are genuine mechanisms versus inherited correlations from a shared cause. | |
| Return ONLY a JSON object of exactly this form, with no text before or after it: | |
| {{"effect": <signed number>, "reasoning": "<two or three sentences>"}} | |
| """ | |
| # -------------------------------------------------------------------------------------- | |
| # Model plumbing | |
| # -------------------------------------------------------------------------------------- | |
| def load_model(repo_id: str): | |
| from transformers import AutoModelForCausalLM, AutoTokenizer | |
| tok = AutoTokenizer.from_pretrained(repo_id) | |
| tok.padding_side = "left" | |
| if tok.pad_token is None: | |
| tok.pad_token = tok.eos_token | |
| model = AutoModelForCausalLM.from_pretrained( | |
| repo_id, dtype=torch.bfloat16, device_map="cuda" | |
| ) | |
| model.eval() | |
| return tok, model | |
| def _chat(tok, user_msg: str, thinking: bool) -> str: | |
| msgs = [{"role": "user", "content": user_msg}] | |
| try: | |
| return tok.apply_chat_template( | |
| msgs, tokenize=False, add_generation_prompt=True, enable_thinking=thinking | |
| ) | |
| except TypeError: | |
| return tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True) | |
| def generate(tok, model, prompts: List[str], max_new_tokens: int = 512, batch_size: int = 8) -> List[str]: | |
| outs: List[str] = [] | |
| for i in range(0, len(prompts), batch_size): | |
| chunk = prompts[i : i + batch_size] | |
| enc = tok(chunk, return_tensors="pt", padding=True, truncation=True, max_length=4096).to( | |
| model.device | |
| ) | |
| gen = model.generate( | |
| **enc, | |
| max_new_tokens=max_new_tokens, | |
| do_sample=False, | |
| temperature=None, | |
| top_p=None, | |
| top_k=None, | |
| pad_token_id=tok.pad_token_id, | |
| ) | |
| new = gen[:, enc["input_ids"].shape[1] :] | |
| outs.extend(tok.batch_decode(new, skip_special_tokens=True)) | |
| print(f" ...{min(i + batch_size, len(prompts))}/{len(prompts)}", flush=True) | |
| return outs | |
| # -------------------------------------------------------------------------------------- | |
| # Parsing | |
| # -------------------------------------------------------------------------------------- | |
| def _first_json(text: str): | |
| m = re.search(r"\{.*?\}", text, re.S) | |
| if not m: | |
| return None | |
| try: | |
| return json.loads(m.group(0)) | |
| except Exception: | |
| try: | |
| return json.loads(re.sub(r",\s*([}\]])", r"\1", m.group(0))) | |
| except Exception: | |
| return None | |
| def parse_names(text: str, key: str, allowed: List[str]) -> List[str]: | |
| obj = _first_json(text) | |
| names: List[str] = [] | |
| if isinstance(obj, dict) and isinstance(obj.get(key), list): | |
| for v in obj[key]: | |
| if isinstance(v, str): | |
| v = v.strip().strip("`").lower().replace(" ", "_") | |
| if v in allowed and v not in names: | |
| names.append(v) | |
| if not names: # fallback: scan for known names in the raw text | |
| low = text.lower() | |
| for n in allowed: | |
| if n in low and n not in names: | |
| names.append(n) | |
| return names | |
| def parse_effect(text: str): | |
| obj = _first_json(text) | |
| if isinstance(obj, dict) and "effect" in obj: | |
| try: | |
| return float(obj["effect"]), str(obj.get("reasoning", "")) | |
| except Exception: | |
| pass | |
| m = re.search(r'"effect"\s*:\s*(-?\d+(?:\.\d+)?)', text) | |
| if m: | |
| return float(m.group(1)), "" | |
| m = re.search(r"(-?\d+(?:\.\d+)?)", text) | |
| if m: | |
| try: | |
| return float(m.group(1)), "" | |
| except Exception: | |
| return None, "" | |
| return None, "" | |
| # -------------------------------------------------------------------------------------- | |
| # Main | |
| # -------------------------------------------------------------------------------------- | |
| def run_model(model_cfg, instances, out_dir: str, tasks=ALL_TASKS, merge: bool = False): | |
| repo_id = str(model_cfg["id"]) | |
| label = str(model_cfg["label"]) | |
| thinking = bool(model_cfg["thinking"]) | |
| print(f"\n=== {label} ({repo_id}) ===", flush=True) | |
| tok, model = load_model(repo_id) | |
| results: Dict[str, List[dict]] = {t: [] for t in ALL_TASKS} | |
| os.makedirs(out_dir, exist_ok=True) | |
| path = os.path.join(out_dir, f"raw_{label.replace('/', '_')}.json") | |
| if merge and os.path.exists(path): | |
| prev = json.load(open(path, encoding="utf8")) | |
| for k, v in prev.get("results", {}).items(): | |
| results.setdefault(k, []) | |
| results[k] = v | |
| if "select" in tasks: | |
| prompts = [_chat(tok, prompt_select(i), thinking) for i in instances] | |
| t0 = time.time() | |
| outs = generate(tok, model, prompts) | |
| print(f" select done in {time.time()-t0:.0f}s", flush=True) | |
| for inst, out in zip(instances, outs): | |
| results["select"].append( | |
| {"instance": inst.spec.seed, "raw": out, "pred": parse_names(out, "causes", CANDIDATE_NAMES)} | |
| ) | |
| if "predict" in tasks: | |
| prompts = [_chat(tok, prompt_predict(i), thinking) for i in instances] | |
| outs = generate(tok, model, prompts) | |
| for inst, out in zip(instances, outs): | |
| results["predict"].append( | |
| {"instance": inst.spec.seed, "raw": out, "pred": parse_names(out, "predictors", CANDIDATE_NAMES)} | |
| ) | |
| if "intervene" in tasks: | |
| prompts, keys = [], [] | |
| for inst in instances: | |
| for v in CANDIDATE_NAMES: | |
| prompts.append(_chat(tok, prompt_intervene(inst, v), thinking)) | |
| keys.append((inst.spec.seed, v)) | |
| outs = generate(tok, model, prompts) | |
| for (seed, var), out in zip(keys, outs): | |
| eff, _ = parse_effect(out) | |
| results["intervene"].append( | |
| {"instance": seed, "variable": var, "effect": eff, "raw": out} | |
| ) | |
| if "select_trial" in tasks: | |
| prompts = [_chat(tok, prompt_select_trial(i), thinking) for i in instances] | |
| outs = generate(tok, model, prompts) | |
| for inst, out in zip(instances, outs): | |
| results["select_trial"].append( | |
| {"instance": inst.spec.seed, "raw": out, "pred": parse_names(out, "causes", CANDIDATE_NAMES)} | |
| ) | |
| if "intervene_trial" in tasks: | |
| prompts, keys = [], [] | |
| for inst in instances: | |
| for v in CANDIDATE_NAMES: | |
| prompts.append(_chat(tok, prompt_intervene_trial(inst, v), thinking)) | |
| keys.append((inst.spec.seed, v)) | |
| outs = generate(tok, model, prompts) | |
| for (seed, var), out in zip(keys, outs): | |
| eff, _ = parse_effect(out) | |
| results["intervene_trial"].append( | |
| {"instance": seed, "variable": var, "effect": eff, "raw": out} | |
| ) | |
| with open(path, "w", encoding="utf8") as f: | |
| json.dump({"model": repo_id, "label": label, "results": results}, f, indent=1) | |
| print(f" wrote {path}", flush=True) | |
| del model | |
| torch.cuda.empty_cache() | |
| return path | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--models", nargs="*", default=None, help="subset of model labels") | |
| ap.add_argument("--instances", type=int, default=12) | |
| ap.add_argument("--out", default="results") | |
| ap.add_argument("--tasks", nargs="*", default=list(ALL_TASKS)) | |
| ap.add_argument("--merge", action="store_true") | |
| args = ap.parse_args() | |
| instances = make_instances(args.instances) | |
| print(f"generated {len(instances)} instances; tasks={args.tasks}", flush=True) | |
| os.makedirs(args.out, exist_ok=True) | |
| for cfg in MODELS: | |
| if args.models and cfg["label"] not in args.models: | |
| continue | |
| try: | |
| run_model(cfg, instances, args.out, tasks=tuple(args.tasks), merge=args.merge) | |
| except Exception as e: | |
| print(f"!! {cfg['label']} failed: {type(e).__name__}: {e}", flush=True) | |
| import traceback | |
| traceback.print_exc() | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 16.5 kB
- Xet hash:
- ca61eefbdf5a1b81bcfa0293de1a026809193b4dbff91d575da2f4bb861ffa17
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.