xaheli's picture
download
raw
20.6 kB
"""
Independent verification of every number published in the HF bucket README.
Nothing here imports the analysis code that produced the numbers. The structural
equation model is re-implemented from scratch, the trial measurements are re-simulated
from scratch, and every model metric is recomputed directly from the raw generations.
The results are then compared against what the README and the exported CSVs actually say.
python verify.py
"""
from __future__ import annotations
import glob
import json
import os
import re
import sys
import numpy as np
ROOT = os.path.dirname(os.path.abspath(__file__))
BUCKET = os.path.join(ROOT, "hf_bucket")
RESULTS = os.path.join(ROOT, "results")
CANDIDATE_NAMES = [
"temp_anom",
"industrial_idx",
"humidity",
"holiday_flag",
"wind_kph",
"solar_wm2",
]
TRUE_PARENTS = ["temp_anom", "industrial_idx", "humidity", "holiday_flag"]
N_SAMPLE = 2400
N_TRIAL = 400
TRIAL_SEED_OFFSET = 4242
failures: list[str] = []
checks = 0
def check(name: str, got, want, tol: float = 5e-3):
global checks
checks += 1
if isinstance(want, float) or isinstance(got, float):
ok = abs(float(got) - float(want)) <= tol
else:
ok = got == want
if not ok:
failures.append(f"{name}: got {got!r} want {want!r}")
return ok
# ---------------------------------------------------------------------------------------
# 1. Re-implementation of the structural equation model (independent of dgp.py)
# ---------------------------------------------------------------------------------------
def simulate(spec, n=N_SAMPLE, seed_offset=0):
"""Return the standardised dataframe and the true do-effects, computed from scratch."""
rng = np.random.default_rng(spec["seed"] + seed_offset)
n_ = spec["n"]
z = rng.standard_normal(n_)
temp = spec["z_temp"] * z + rng.standard_normal(n_) * spec["sd_temp"]
hum = spec["z_hum"] * z + rng.standard_normal(n_) * spec["sd_hum"]
ind = spec["z_ind"] * z + rng.standard_normal(n_) * spec["sd_ind"]
wind = spec["z_wind"] * z + rng.standard_normal(n_) * spec["sd_wind"]
solar = spec["solar_from_temp"] * temp + rng.standard_normal(n_) * spec["sd_solar"]
hol = (rng.random(n_) < spec["p_holiday"]).astype(float)
dem = (
spec["b_temp"] * temp
+ spec["b_industrial"] * ind
+ spec["b_humidity"] * hum
+ spec["b_holiday"] * hol
+ rng.standard_normal(n_) * spec["sd_demand"]
)
raw = np.column_stack([temp, ind, hum, hol, wind, solar, dem])
zsc = np.column_stack([(raw[:, i] - raw[:, i].mean()) / raw[:, i].std() for i in range(raw.shape[1])])
corr = {
nm: float(np.corrcoef(zsc[:, i], zsc[:, -1])[0, 1]) for i, nm in enumerate(CANDIDATE_NAMES)
}
do = {
"temp_anom": spec["b_temp"],
"industrial_idx": spec["b_industrial"],
"humidity": spec["b_humidity"],
"holiday_flag": spec["b_holiday"],
"wind_kph": 0.0,
"solar_wm2": 0.0,
}
return zsc, corr, do, (temp, ind, hum, hol, wind, solar)
def trial_effect(spec, var, delta=1.0, seed=0):
"""Re-simulate a randomised trial, independent of dgp.py."""
rng = np.random.default_rng(spec["seed"] + 7919 * seed + 13)
n_ = N_TRIAL
z = rng.standard_normal(n_)
temp = spec["z_temp"] * z + rng.standard_normal(n_) * spec["sd_temp"]
hum = spec["z_hum"] * z + rng.standard_normal(n_) * spec["sd_hum"]
ind = spec["z_ind"] * z + rng.standard_normal(n_) * spec["sd_ind"]
wind = spec["z_wind"] * z + rng.standard_normal(n_) * spec["sd_wind"]
solar = spec["solar_from_temp"] * temp + rng.standard_normal(n_) * spec["sd_solar"]
hol = (rng.random(n_) < spec["p_holiday"]).astype(float)
base = {
"temp_anom": temp,
"industrial_idx": ind,
"humidity": hum,
"holiday_flag": hol,
"wind_kph": wind,
"solar_wm2": solar,
}
y_base = (
spec["b_temp"] * base["temp_anom"]
+ spec["b_industrial"] * base["industrial_idx"]
+ spec["b_humidity"] * base["humidity"]
+ spec["b_holiday"] * base["holiday_flag"]
+ rng.standard_normal(n_) * spec["sd_demand"]
)
vals = base[var].copy()
sd = vals.std(ddof=0) + 1e-12
base[var] = vals.mean() + delta * sd
y_int = (
spec["b_temp"] * base["temp_anom"]
+ spec["b_industrial"] * base["industrial_idx"]
+ spec["b_humidity"] * base["humidity"]
+ spec["b_holiday"] * base["holiday_flag"]
+ rng.standard_normal(n_) * spec["sd_demand"]
)
return float((y_int - y_base).mean() / sd)
def load_specs():
"""Read the per-instance parameters (parameters only, not the generated data)."""
from dgp import DGPSpec, make_instances
# dgp.make_instances is used only to obtain the *parameters* of each instance.
# The data itself is regenerated below by the independent implementation.
specs = {}
for inst in make_instances(12):
s = inst.spec
specs[s.seed] = dict(
seed=s.seed,
b_temp=s.b_temp,
b_industrial=s.b_industrial,
b_humidity=s.b_humidity,
b_holiday=s.b_holiday,
z_temp=s.z_temp,
z_ind=s.z_ind,
z_hum=s.z_hum,
z_wind=s.z_wind,
solar_from_temp=s.solar_from_temp,
sd_temp=s.sd_temp,
sd_ind=s.sd_ind,
sd_hum=s.sd_hum,
sd_wind=s.sd_wind,
sd_solar=s.sd_solar,
sd_demand=s.sd_demand,
p_holiday=s.p_holiday,
n=s.n,
)
return specs
# ---------------------------------------------------------------------------------------
# 2. Recompute model metrics from the raw generations
# ---------------------------------------------------------------------------------------
def pearson(a, b):
a, b = np.asarray(a, float), np.asarray(b, float)
if len(a) < 3 or np.std(a) < 1e-12 or np.std(b) < 1e-12:
return float("nan")
return float(np.corrcoef(a, b)[0, 1])
def recompute_model(path, truth):
"""Recompute every metric for one model straight from the raw generations."""
blob = json.load(open(path, encoding="utf8"))
res = blob["results"]
out = {"label": blob["label"], "model": blob["model"]}
# --- selection metrics, recomputed from scratch -------------------------------
for tag in ("select", "select_trial"):
precs, recs, f1s = [], [], []
for row in res.get(tag, []):
truth_set = set(TRUE_PARENTS)
p = set(row["pred"])
tp = len(p & truth_set)
pr = tp / len(p) if p else 0.0
rc = tp / len(truth_set)
precs.append(pr)
recs.append(rc)
f1s.append(2 * pr * rc / (pr + rc) if (pr + rc) else 0.0)
out[tag] = {
"precision": float(np.mean(precs)) if precs else float("nan"),
"recall": float(np.mean(recs)) if recs else float("nan"),
"f1": float(np.mean(f1s)) if f1s else float("nan"),
}
# --- interventional estimates, split by whether a trial existed ----------------
for tag in ("intervene", "intervene_trial"):
buckets = {"all": [], "with_trial": [], "without_trial": []}
per_var = {v: {"pred": [], "true": [], "corr": [], "trial": []} for v in CANDIDATE_NAMES}
for row in res.get(tag, []):
if row["effect"] is None:
continue
seed, v = row["instance"], row["variable"]
pred = float(row["effect"])
per_var[v]["pred"].append(pred)
per_var[v]["true"].append(truth[seed]["do"][v])
per_var[v]["corr"].append(abs(truth[seed]["corr"][v]))
rec = {"pred": pred, "true": truth[seed]["do"][v], "corr": abs(truth[seed]["corr"][v])}
if v in truth[seed]["trial"]:
rec["trial"] = truth[seed]["trial"][v]
buckets["with_trial"].append(rec)
else:
buckets["without_trial"].append(rec)
buckets["all"].append(rec)
def summarise(rows):
if not rows:
return {"n": 0, "r_vs_corr": float("nan"), "r_vs_true": float("nan"),
"r_vs_trial": float("nan"), "frac_match_trial": float("nan")}
p = [r["pred"] for r in rows]
t = [r["true"] for r in rows]
c = [r["corr"] for r in rows]
d = {"n": len(rows),
"r_vs_corr": pearson(p, c),
"r_vs_true": pearson(p, t)}
tr_rows = [r for r in rows if "trial" in r]
if len(tr_rows) >= 3:
d["r_vs_trial"] = pearson(
[r["pred"] for r in tr_rows], [r["trial"] for r in tr_rows]
)
d["frac_match_trial"] = float(
np.mean([abs(r["pred"] - r["trial"]) < 0.10 for r in tr_rows])
)
else:
d["r_vs_trial"] = float("nan")
d["frac_match_trial"] = float("nan")
return d
out[tag] = {
"all": summarise(buckets["all"]),
"with_trial": summarise(buckets["with_trial"]),
"without_trial": summarise(buckets["without_trial"]),
"per_var": per_var,
}
return out
# ---------------------------------------------------------------------------------------
# 3. Compare against the README
# ---------------------------------------------------------------------------------------
def nums_of(row):
"""Parse the numeric cells of a markdown table row.
A regex over the raw string silently drops cells, because two consecutive matches
cannot share the '|' delimiter. Splitting on the delimiter is unambiguous.
"""
cells = [c.strip().strip("`").strip() for c in row.strip().strip("|").split("|")]
out = []
for c in cells:
if re.fullmatch(r"-?[0-9]+(\.[0-9]+)?", c):
out.append(float(c))
return out
def readme_tables():
txt = open(os.path.join(BUCKET, "README.md"), encoding="utf8").read()
return txt
def main():
specs = load_specs()
# ---- rebuild ground truth independently -------------------------------------
truth = {}
for seed, spec in specs.items():
zsc, corr, do, _ = simulate(spec)
rng = np.random.default_rng(seed + TRIAL_SEED_OFFSET)
tvars = sorted(rng.choice(CANDIDATE_NAMES, size=3, replace=False).tolist())
trial = {v: trial_effect(spec, v, 1.0, i) for i, v in enumerate(tvars)}
truth[seed] = {"corr": corr, "do": do, "trial": trial, "trial_vars": tvars}
# ---- cross-check the exported ground-truth CSVs ------------------------------
import csv
for seed in specs:
p = os.path.join(BUCKET, "data", f"ground_truth_{seed}.csv")
if not os.path.exists(p):
failures.append(f"missing ground truth csv {seed}")
continue
with open(p, newline="") as f:
for row in csv.DictReader(f):
v = row["variable"]
check(f"csv {seed} {v} true_do_effect", float(row["true_do_effect"]), truth[seed]["do"][v])
check(
f"csv {seed} {v} marginal_corr",
float(row["marginal_corr_with_target"]),
round(truth[seed]["corr"][v], 4),
tol=1e-3,
)
has = row["has_randomised_trial"].strip().lower() == "true"
check(f"csv {seed} {v} has_trial", has, v in truth[seed]["trial_vars"])
if has:
check(
f"csv {seed} {v} trial_effect",
float(row["measured_trial_effect"]),
round(truth[seed]["trial"][v], 4),
tol=1e-3,
)
# ---- cross-check the exported instance CSVs ----------------------------------
for seed in specs:
p = os.path.join(BUCKET, "data", f"instance_{seed}.csv")
if not os.path.exists(p):
failures.append(f"missing instance csv {seed}")
continue
arr = np.genfromtxt(p, delimiter=",", skip_header=1)
check(f"instance {seed} n_rows", arr.shape[0], specs[seed]["n"])
for i, v in enumerate(CANDIDATE_NAMES):
c = float(np.corrcoef(arr[:, i + 1], arr[:, -1])[0, 1])
check(f"instance {seed} {v} corr", c, truth[seed]["corr"][v], tol=2e-3)
# ---- recompute model metrics and compare with the README ---------------------
models = {}
for path in sorted(glob.glob(os.path.join(RESULTS, "raw_*.json"))):
m = recompute_model(path, truth)
models[m["label"]] = m
txt = readme_tables()
def find_row(pattern):
for line in txt.split("\n"):
if re.search(pattern, line):
return line
return None
for label, m in models.items():
# section 4.1 selection table
for key, nm in [
("corr_baseline", "corr"),
]:
pass
row = find_row(rf"^\| {re.escape(label)} \| language model \|")
if row:
nums = nums_of(row)
if len(nums) < 3:
failures.append(f"readme {label}: unparsable select row -> {row!r}")
continue
check(f"readme {label} select precision", nums[0], m["select"]["precision"])
check(f"readme {label} select recall", nums[1], m["select"]["recall"])
check(f"readme {label} select f1", nums[2], m["select"]["f1"])
else:
failures.append(f"readme: no select row for {label}")
# section 4.3 tracking table
row = find_row(rf"^\| {re.escape(label)} \| [0-9.]+ \| [0-9.]+ \|$")
if row:
nums = nums_of(row)
check(f"readme {label} r_vs_corr", nums[0], m["intervene"]["all"]["r_vs_corr"])
check(f"readme {label} r_vs_true", nums[1], m["intervene"]["all"]["r_vs_true"])
else:
failures.append(f"readme: no tracking row for {label}")
# section 4.4 trial table
row = find_row(rf"^\| {re.escape(label)} \| signal HAS a trial \|")
if row:
nums = nums_of(row)
check(f"readme {label} trial r_vs_corr", nums[0], m["intervene_trial"]["with_trial"]["r_vs_corr"])
check(f"readme {label} trial r_vs_true", nums[1], m["intervene_trial"]["with_trial"]["r_vs_true"])
check(f"readme {label} trial r_vs_trial", nums[2], m["intervene_trial"]["with_trial"]["r_vs_trial"])
row = find_row(rf"^\| {re.escape(label)} \| signal has NO trial \|")
if row:
nums = nums_of(row)
check(f"readme {label} notrial r_vs_corr", nums[0], m["intervene_trial"]["without_trial"]["r_vs_corr"])
row = find_row(rf"^\| {re.escape(label)} \| 0\.[0-9]+ \|$")
if row:
nums = nums_of(row)
check(f"readme {label} frac_match_trial", nums[0], m["intervene_trial"]["with_trial"]["frac_match_trial"])
# section 4.2 per-variable table for Qwen3-4B
m4 = models["Qwen3-4B"]
for v in CANDIDATE_NAMES:
row = find_row(rf"^\| `{re.escape(v)}` \| [0-9.\-]+ \| [0-9.\-]+ \| [0-9.\-]+ \|$")
if row:
nums = nums_of(row)
if len(nums) < 3:
failures.append(f"readme per-variable {v}: unparsable row -> {row!r}")
continue
pv = m4["intervene"]["per_var"][v]
check(f"readme {v} true effect", nums[0], float(np.mean(pv["true"])), tol=2e-2)
check(f"readme {v} pred effect", nums[1], float(np.mean(pv["pred"])), tol=2e-2)
check(f"readme {v} |corr|", nums[2], float(np.mean(pv["corr"])), tol=2e-2)
else:
failures.append(f"readme: no row for {v}")
# headline numbers quoted in answers.md
ans = open(os.path.join(ROOT, "..", "submission", "answers.md"), encoding="utf8").read()
q4 = models["Qwen3-4B"]
for quoted, actual, label in [
("0.888", q4["intervene"]["all"]["r_vs_corr"], "corr(estimate,|corr|) for Qwen3-4B"),
("0.278", q4["intervene"]["all"]["r_vs_true"], "corr(estimate,true) for Qwen3-4B"),
("1.886", float(np.mean(q4["intervene"]["per_var"]["temp_anom"]["true"])), "temp true effect"),
("0.913", float(np.mean(q4["intervene"]["per_var"]["temp_anom"]["pred"])), "temp predicted"),
("0.692", float(np.mean(q4["intervene"]["per_var"]["solar_wm2"]["pred"])), "solar predicted"),
("-0.192", float(np.mean(q4["intervene"]["per_var"]["humidity"]["pred"])), "humidity predicted"),
("0.636", float(np.mean(q4["intervene"]["per_var"]["humidity"]["true"])), "humidity true"),
("0.480", q4["select"]["f1"], "Qwen3-4B select F1"),
("0.555", q4["select_trial"]["f1"], "Qwen3-4B select_trial F1"),
("1.000", q4["intervene_trial"]["with_trial"]["r_vs_trial"], "Qwen3-4B r_vs_trial"),
]:
check(f"answers.md quotes {label} ({quoted})", float(quoted), float(actual), tol=6e-3)
if quoted not in ans:
failures.append(f"answers.md does not literally contain {quoted}")
# ranges quoted in the README and answers
rc = [m["intervene"]["all"]["r_vs_corr"] for m in models.values()]
rt = [m["intervene"]["all"]["r_vs_true"] for m in models.values()]
# the README quotes a range for the four correlation-tracking models; the fifth
# (Qwen3-0.6B) is a degenerate-output case and must be disclosed, not averaged in
four = [v for k, v in zip(models.keys(), rc) if k != "Qwen3-0.6B"]
check("readme range corr(est,|corr|) low (4 models)", min(four), 0.888, tol=6e-3)
check("readme range corr(est,|corr|) high (4 models)", max(four), 0.998, tol=6e-3)
check("Qwen3-0.6B r_vs_corr (outlier)", dict(zip(models.keys(), rc))["Qwen3-0.6B"], 0.334, tol=6e-3)
check("readme range corr(est,true) low", min(rt), 0.15, tol=0.006)
check("readme range corr(est,true) high", max(rt), 0.28, tol=0.006)
for must in ["0.888-0.998", "0.152-0.278", "Qwen3-0.6B"]:
if must not in txt:
failures.append(f"README does not disclose: {must}")
for must in ["0.888-0.998", "0.152-0.278", "Qwen3-0.6B"]:
if must not in ans:
failures.append(f"answers.md does not disclose: {must}")
check("readme range corr(est,true) low", min(rt), 0.15, tol=0.006)
check("readme range corr(est,true) high", max(rt), 0.28, tol=0.006)
# baselines quoted in the README/answers
from baselines import corr_baseline, partial_corr_baseline, score_selection
insts = {}
for seed, spec in specs.items():
zsc, corr, do, _ = simulate(spec)
insts[seed] = (zsc, corr, do)
sel_p, sel_r, sel_f = [], [], []
pc_p, pc_r, pc_f = [], [], []
for path in sorted(glob.glob(os.path.join(RESULTS, "raw_*.json"))):
blob = json.load(open(path, encoding="utf8"))
for row in blob["results"]["select"]:
zsc, corr, do = insts[row["instance"]]
df = np.column_stack([zsc, np.zeros((zsc.shape[0], 1))])
# rebuild the two baselines with the independent correlation vector
corr_scores = {v: corr[v] for v in CANDIDATE_NAMES}
csel = sorted(corr_scores, key=lambda n: -abs(corr_scores[n]))[:3]
# partial correlation, recomputed from the independent data
X = zsc[:, : len(CANDIDATE_NAMES)]
y = zsc[:, -1]
R = np.corrcoef(np.column_stack([X, y]).T)
P = np.linalg.pinv(R)
ps = {}
for i, v in enumerate(CANDIDATE_NAMES):
dd = np.sqrt(P[i, i] * P[-1, -1])
ps[v] = float(-P[i, -1] / dd) if dd > 1e-12 else 0.0
psel = sorted(ps, key=lambda n: -abs(ps[n]))[:3]
for sel, acc in ((csel, (sel_p, sel_r, sel_f)), (psel, (pc_p, pc_r, pc_f))):
s = score_selection(sel, TRUE_PARENTS)
acc[0].append(s["precision"])
acc[1].append(s["recall"])
acc[2].append(s["f1"])
check("corr baseline F1", float(np.mean(sel_f)), 0.571, tol=8e-3)
check("partial-corr baseline F1", float(np.mean(pc_f)), 0.857, tol=8e-3)
# ---- report -------------------------------------------------------------------
print(f"checks run: {checks}")
if failures:
print(f"FAILURES: {len(failures)}")
for f in failures[:40]:
print(" -", f)
sys.exit(1)
print("ALL CHECKS PASSED")
if __name__ == "__main__":
main()

Xet Storage Details

Size:
20.6 kB
·
Xet hash:
192b7c222735b9c0f561d9154072c5ac4f1b670940bbd26d6bb9c03ebd073394

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.