Buckets:
| """ | |
| Independent verification of every number published in the HF bucket README. | |
| Nothing here imports the analysis code that produced the numbers. The structural | |
| equation model is re-implemented from scratch, the trial measurements are re-simulated | |
| from scratch, and every model metric is recomputed directly from the raw generations. | |
| The results are then compared against what the README and the exported CSVs actually say. | |
| python verify.py | |
| """ | |
| from __future__ import annotations | |
| import glob | |
| import json | |
| import os | |
| import re | |
| import sys | |
| import numpy as np | |
| ROOT = os.path.dirname(os.path.abspath(__file__)) | |
| BUCKET = os.path.join(ROOT, "hf_bucket") | |
| RESULTS = os.path.join(ROOT, "results") | |
| CANDIDATE_NAMES = [ | |
| "temp_anom", | |
| "industrial_idx", | |
| "humidity", | |
| "holiday_flag", | |
| "wind_kph", | |
| "solar_wm2", | |
| ] | |
| TRUE_PARENTS = ["temp_anom", "industrial_idx", "humidity", "holiday_flag"] | |
| N_SAMPLE = 2400 | |
| N_TRIAL = 400 | |
| TRIAL_SEED_OFFSET = 4242 | |
| failures: list[str] = [] | |
| checks = 0 | |
| def check(name: str, got, want, tol: float = 5e-3): | |
| global checks | |
| checks += 1 | |
| if isinstance(want, float) or isinstance(got, float): | |
| ok = abs(float(got) - float(want)) <= tol | |
| else: | |
| ok = got == want | |
| if not ok: | |
| failures.append(f"{name}: got {got!r} want {want!r}") | |
| return ok | |
| # --------------------------------------------------------------------------------------- | |
| # 1. Re-implementation of the structural equation model (independent of dgp.py) | |
| # --------------------------------------------------------------------------------------- | |
| def simulate(spec, n=N_SAMPLE, seed_offset=0): | |
| """Return the standardised dataframe and the true do-effects, computed from scratch.""" | |
| rng = np.random.default_rng(spec["seed"] + seed_offset) | |
| n_ = spec["n"] | |
| z = rng.standard_normal(n_) | |
| temp = spec["z_temp"] * z + rng.standard_normal(n_) * spec["sd_temp"] | |
| hum = spec["z_hum"] * z + rng.standard_normal(n_) * spec["sd_hum"] | |
| ind = spec["z_ind"] * z + rng.standard_normal(n_) * spec["sd_ind"] | |
| wind = spec["z_wind"] * z + rng.standard_normal(n_) * spec["sd_wind"] | |
| solar = spec["solar_from_temp"] * temp + rng.standard_normal(n_) * spec["sd_solar"] | |
| hol = (rng.random(n_) < spec["p_holiday"]).astype(float) | |
| dem = ( | |
| spec["b_temp"] * temp | |
| + spec["b_industrial"] * ind | |
| + spec["b_humidity"] * hum | |
| + spec["b_holiday"] * hol | |
| + rng.standard_normal(n_) * spec["sd_demand"] | |
| ) | |
| raw = np.column_stack([temp, ind, hum, hol, wind, solar, dem]) | |
| zsc = np.column_stack([(raw[:, i] - raw[:, i].mean()) / raw[:, i].std() for i in range(raw.shape[1])]) | |
| corr = { | |
| nm: float(np.corrcoef(zsc[:, i], zsc[:, -1])[0, 1]) for i, nm in enumerate(CANDIDATE_NAMES) | |
| } | |
| do = { | |
| "temp_anom": spec["b_temp"], | |
| "industrial_idx": spec["b_industrial"], | |
| "humidity": spec["b_humidity"], | |
| "holiday_flag": spec["b_holiday"], | |
| "wind_kph": 0.0, | |
| "solar_wm2": 0.0, | |
| } | |
| return zsc, corr, do, (temp, ind, hum, hol, wind, solar) | |
| def trial_effect(spec, var, delta=1.0, seed=0): | |
| """Re-simulate a randomised trial, independent of dgp.py.""" | |
| rng = np.random.default_rng(spec["seed"] + 7919 * seed + 13) | |
| n_ = N_TRIAL | |
| z = rng.standard_normal(n_) | |
| temp = spec["z_temp"] * z + rng.standard_normal(n_) * spec["sd_temp"] | |
| hum = spec["z_hum"] * z + rng.standard_normal(n_) * spec["sd_hum"] | |
| ind = spec["z_ind"] * z + rng.standard_normal(n_) * spec["sd_ind"] | |
| wind = spec["z_wind"] * z + rng.standard_normal(n_) * spec["sd_wind"] | |
| solar = spec["solar_from_temp"] * temp + rng.standard_normal(n_) * spec["sd_solar"] | |
| hol = (rng.random(n_) < spec["p_holiday"]).astype(float) | |
| base = { | |
| "temp_anom": temp, | |
| "industrial_idx": ind, | |
| "humidity": hum, | |
| "holiday_flag": hol, | |
| "wind_kph": wind, | |
| "solar_wm2": solar, | |
| } | |
| y_base = ( | |
| spec["b_temp"] * base["temp_anom"] | |
| + spec["b_industrial"] * base["industrial_idx"] | |
| + spec["b_humidity"] * base["humidity"] | |
| + spec["b_holiday"] * base["holiday_flag"] | |
| + rng.standard_normal(n_) * spec["sd_demand"] | |
| ) | |
| vals = base[var].copy() | |
| sd = vals.std(ddof=0) + 1e-12 | |
| base[var] = vals.mean() + delta * sd | |
| y_int = ( | |
| spec["b_temp"] * base["temp_anom"] | |
| + spec["b_industrial"] * base["industrial_idx"] | |
| + spec["b_humidity"] * base["humidity"] | |
| + spec["b_holiday"] * base["holiday_flag"] | |
| + rng.standard_normal(n_) * spec["sd_demand"] | |
| ) | |
| return float((y_int - y_base).mean() / sd) | |
| def load_specs(): | |
| """Read the per-instance parameters (parameters only, not the generated data).""" | |
| from dgp import DGPSpec, make_instances | |
| # dgp.make_instances is used only to obtain the *parameters* of each instance. | |
| # The data itself is regenerated below by the independent implementation. | |
| specs = {} | |
| for inst in make_instances(12): | |
| s = inst.spec | |
| specs[s.seed] = dict( | |
| seed=s.seed, | |
| b_temp=s.b_temp, | |
| b_industrial=s.b_industrial, | |
| b_humidity=s.b_humidity, | |
| b_holiday=s.b_holiday, | |
| z_temp=s.z_temp, | |
| z_ind=s.z_ind, | |
| z_hum=s.z_hum, | |
| z_wind=s.z_wind, | |
| solar_from_temp=s.solar_from_temp, | |
| sd_temp=s.sd_temp, | |
| sd_ind=s.sd_ind, | |
| sd_hum=s.sd_hum, | |
| sd_wind=s.sd_wind, | |
| sd_solar=s.sd_solar, | |
| sd_demand=s.sd_demand, | |
| p_holiday=s.p_holiday, | |
| n=s.n, | |
| ) | |
| return specs | |
| # --------------------------------------------------------------------------------------- | |
| # 2. Recompute model metrics from the raw generations | |
| # --------------------------------------------------------------------------------------- | |
| def pearson(a, b): | |
| a, b = np.asarray(a, float), np.asarray(b, float) | |
| if len(a) < 3 or np.std(a) < 1e-12 or np.std(b) < 1e-12: | |
| return float("nan") | |
| return float(np.corrcoef(a, b)[0, 1]) | |
| def recompute_model(path, truth): | |
| """Recompute every metric for one model straight from the raw generations.""" | |
| blob = json.load(open(path, encoding="utf8")) | |
| res = blob["results"] | |
| out = {"label": blob["label"], "model": blob["model"]} | |
| # --- selection metrics, recomputed from scratch ------------------------------- | |
| for tag in ("select", "select_trial"): | |
| precs, recs, f1s = [], [], [] | |
| for row in res.get(tag, []): | |
| truth_set = set(TRUE_PARENTS) | |
| p = set(row["pred"]) | |
| tp = len(p & truth_set) | |
| pr = tp / len(p) if p else 0.0 | |
| rc = tp / len(truth_set) | |
| precs.append(pr) | |
| recs.append(rc) | |
| f1s.append(2 * pr * rc / (pr + rc) if (pr + rc) else 0.0) | |
| out[tag] = { | |
| "precision": float(np.mean(precs)) if precs else float("nan"), | |
| "recall": float(np.mean(recs)) if recs else float("nan"), | |
| "f1": float(np.mean(f1s)) if f1s else float("nan"), | |
| } | |
| # --- interventional estimates, split by whether a trial existed ---------------- | |
| for tag in ("intervene", "intervene_trial"): | |
| buckets = {"all": [], "with_trial": [], "without_trial": []} | |
| per_var = {v: {"pred": [], "true": [], "corr": [], "trial": []} for v in CANDIDATE_NAMES} | |
| for row in res.get(tag, []): | |
| if row["effect"] is None: | |
| continue | |
| seed, v = row["instance"], row["variable"] | |
| pred = float(row["effect"]) | |
| per_var[v]["pred"].append(pred) | |
| per_var[v]["true"].append(truth[seed]["do"][v]) | |
| per_var[v]["corr"].append(abs(truth[seed]["corr"][v])) | |
| rec = {"pred": pred, "true": truth[seed]["do"][v], "corr": abs(truth[seed]["corr"][v])} | |
| if v in truth[seed]["trial"]: | |
| rec["trial"] = truth[seed]["trial"][v] | |
| buckets["with_trial"].append(rec) | |
| else: | |
| buckets["without_trial"].append(rec) | |
| buckets["all"].append(rec) | |
| def summarise(rows): | |
| if not rows: | |
| return {"n": 0, "r_vs_corr": float("nan"), "r_vs_true": float("nan"), | |
| "r_vs_trial": float("nan"), "frac_match_trial": float("nan")} | |
| p = [r["pred"] for r in rows] | |
| t = [r["true"] for r in rows] | |
| c = [r["corr"] for r in rows] | |
| d = {"n": len(rows), | |
| "r_vs_corr": pearson(p, c), | |
| "r_vs_true": pearson(p, t)} | |
| tr_rows = [r for r in rows if "trial" in r] | |
| if len(tr_rows) >= 3: | |
| d["r_vs_trial"] = pearson( | |
| [r["pred"] for r in tr_rows], [r["trial"] for r in tr_rows] | |
| ) | |
| d["frac_match_trial"] = float( | |
| np.mean([abs(r["pred"] - r["trial"]) < 0.10 for r in tr_rows]) | |
| ) | |
| else: | |
| d["r_vs_trial"] = float("nan") | |
| d["frac_match_trial"] = float("nan") | |
| return d | |
| out[tag] = { | |
| "all": summarise(buckets["all"]), | |
| "with_trial": summarise(buckets["with_trial"]), | |
| "without_trial": summarise(buckets["without_trial"]), | |
| "per_var": per_var, | |
| } | |
| return out | |
| # --------------------------------------------------------------------------------------- | |
| # 3. Compare against the README | |
| # --------------------------------------------------------------------------------------- | |
| def nums_of(row): | |
| """Parse the numeric cells of a markdown table row. | |
| A regex over the raw string silently drops cells, because two consecutive matches | |
| cannot share the '|' delimiter. Splitting on the delimiter is unambiguous. | |
| """ | |
| cells = [c.strip().strip("`").strip() for c in row.strip().strip("|").split("|")] | |
| out = [] | |
| for c in cells: | |
| if re.fullmatch(r"-?[0-9]+(\.[0-9]+)?", c): | |
| out.append(float(c)) | |
| return out | |
| def readme_tables(): | |
| txt = open(os.path.join(BUCKET, "README.md"), encoding="utf8").read() | |
| return txt | |
| def main(): | |
| specs = load_specs() | |
| # ---- rebuild ground truth independently ------------------------------------- | |
| truth = {} | |
| for seed, spec in specs.items(): | |
| zsc, corr, do, _ = simulate(spec) | |
| rng = np.random.default_rng(seed + TRIAL_SEED_OFFSET) | |
| tvars = sorted(rng.choice(CANDIDATE_NAMES, size=3, replace=False).tolist()) | |
| trial = {v: trial_effect(spec, v, 1.0, i) for i, v in enumerate(tvars)} | |
| truth[seed] = {"corr": corr, "do": do, "trial": trial, "trial_vars": tvars} | |
| # ---- cross-check the exported ground-truth CSVs ------------------------------ | |
| import csv | |
| for seed in specs: | |
| p = os.path.join(BUCKET, "data", f"ground_truth_{seed}.csv") | |
| if not os.path.exists(p): | |
| failures.append(f"missing ground truth csv {seed}") | |
| continue | |
| with open(p, newline="") as f: | |
| for row in csv.DictReader(f): | |
| v = row["variable"] | |
| check(f"csv {seed} {v} true_do_effect", float(row["true_do_effect"]), truth[seed]["do"][v]) | |
| check( | |
| f"csv {seed} {v} marginal_corr", | |
| float(row["marginal_corr_with_target"]), | |
| round(truth[seed]["corr"][v], 4), | |
| tol=1e-3, | |
| ) | |
| has = row["has_randomised_trial"].strip().lower() == "true" | |
| check(f"csv {seed} {v} has_trial", has, v in truth[seed]["trial_vars"]) | |
| if has: | |
| check( | |
| f"csv {seed} {v} trial_effect", | |
| float(row["measured_trial_effect"]), | |
| round(truth[seed]["trial"][v], 4), | |
| tol=1e-3, | |
| ) | |
| # ---- cross-check the exported instance CSVs ---------------------------------- | |
| for seed in specs: | |
| p = os.path.join(BUCKET, "data", f"instance_{seed}.csv") | |
| if not os.path.exists(p): | |
| failures.append(f"missing instance csv {seed}") | |
| continue | |
| arr = np.genfromtxt(p, delimiter=",", skip_header=1) | |
| check(f"instance {seed} n_rows", arr.shape[0], specs[seed]["n"]) | |
| for i, v in enumerate(CANDIDATE_NAMES): | |
| c = float(np.corrcoef(arr[:, i + 1], arr[:, -1])[0, 1]) | |
| check(f"instance {seed} {v} corr", c, truth[seed]["corr"][v], tol=2e-3) | |
| # ---- recompute model metrics and compare with the README --------------------- | |
| models = {} | |
| for path in sorted(glob.glob(os.path.join(RESULTS, "raw_*.json"))): | |
| m = recompute_model(path, truth) | |
| models[m["label"]] = m | |
| txt = readme_tables() | |
| def find_row(pattern): | |
| for line in txt.split("\n"): | |
| if re.search(pattern, line): | |
| return line | |
| return None | |
| for label, m in models.items(): | |
| # section 4.1 selection table | |
| for key, nm in [ | |
| ("corr_baseline", "corr"), | |
| ]: | |
| pass | |
| row = find_row(rf"^\| {re.escape(label)} \| language model \|") | |
| if row: | |
| nums = nums_of(row) | |
| if len(nums) < 3: | |
| failures.append(f"readme {label}: unparsable select row -> {row!r}") | |
| continue | |
| check(f"readme {label} select precision", nums[0], m["select"]["precision"]) | |
| check(f"readme {label} select recall", nums[1], m["select"]["recall"]) | |
| check(f"readme {label} select f1", nums[2], m["select"]["f1"]) | |
| else: | |
| failures.append(f"readme: no select row for {label}") | |
| # section 4.3 tracking table | |
| row = find_row(rf"^\| {re.escape(label)} \| [0-9.]+ \| [0-9.]+ \|$") | |
| if row: | |
| nums = nums_of(row) | |
| check(f"readme {label} r_vs_corr", nums[0], m["intervene"]["all"]["r_vs_corr"]) | |
| check(f"readme {label} r_vs_true", nums[1], m["intervene"]["all"]["r_vs_true"]) | |
| else: | |
| failures.append(f"readme: no tracking row for {label}") | |
| # section 4.4 trial table | |
| row = find_row(rf"^\| {re.escape(label)} \| signal HAS a trial \|") | |
| if row: | |
| nums = nums_of(row) | |
| check(f"readme {label} trial r_vs_corr", nums[0], m["intervene_trial"]["with_trial"]["r_vs_corr"]) | |
| check(f"readme {label} trial r_vs_true", nums[1], m["intervene_trial"]["with_trial"]["r_vs_true"]) | |
| check(f"readme {label} trial r_vs_trial", nums[2], m["intervene_trial"]["with_trial"]["r_vs_trial"]) | |
| row = find_row(rf"^\| {re.escape(label)} \| signal has NO trial \|") | |
| if row: | |
| nums = nums_of(row) | |
| check(f"readme {label} notrial r_vs_corr", nums[0], m["intervene_trial"]["without_trial"]["r_vs_corr"]) | |
| row = find_row(rf"^\| {re.escape(label)} \| 0\.[0-9]+ \|$") | |
| if row: | |
| nums = nums_of(row) | |
| check(f"readme {label} frac_match_trial", nums[0], m["intervene_trial"]["with_trial"]["frac_match_trial"]) | |
| # section 4.2 per-variable table for Qwen3-4B | |
| m4 = models["Qwen3-4B"] | |
| for v in CANDIDATE_NAMES: | |
| row = find_row(rf"^\| `{re.escape(v)}` \| [0-9.\-]+ \| [0-9.\-]+ \| [0-9.\-]+ \|$") | |
| if row: | |
| nums = nums_of(row) | |
| if len(nums) < 3: | |
| failures.append(f"readme per-variable {v}: unparsable row -> {row!r}") | |
| continue | |
| pv = m4["intervene"]["per_var"][v] | |
| check(f"readme {v} true effect", nums[0], float(np.mean(pv["true"])), tol=2e-2) | |
| check(f"readme {v} pred effect", nums[1], float(np.mean(pv["pred"])), tol=2e-2) | |
| check(f"readme {v} |corr|", nums[2], float(np.mean(pv["corr"])), tol=2e-2) | |
| else: | |
| failures.append(f"readme: no row for {v}") | |
| # headline numbers quoted in answers.md | |
| ans = open(os.path.join(ROOT, "..", "submission", "answers.md"), encoding="utf8").read() | |
| q4 = models["Qwen3-4B"] | |
| for quoted, actual, label in [ | |
| ("0.888", q4["intervene"]["all"]["r_vs_corr"], "corr(estimate,|corr|) for Qwen3-4B"), | |
| ("0.278", q4["intervene"]["all"]["r_vs_true"], "corr(estimate,true) for Qwen3-4B"), | |
| ("1.886", float(np.mean(q4["intervene"]["per_var"]["temp_anom"]["true"])), "temp true effect"), | |
| ("0.913", float(np.mean(q4["intervene"]["per_var"]["temp_anom"]["pred"])), "temp predicted"), | |
| ("0.692", float(np.mean(q4["intervene"]["per_var"]["solar_wm2"]["pred"])), "solar predicted"), | |
| ("-0.192", float(np.mean(q4["intervene"]["per_var"]["humidity"]["pred"])), "humidity predicted"), | |
| ("0.636", float(np.mean(q4["intervene"]["per_var"]["humidity"]["true"])), "humidity true"), | |
| ("0.480", q4["select"]["f1"], "Qwen3-4B select F1"), | |
| ("0.555", q4["select_trial"]["f1"], "Qwen3-4B select_trial F1"), | |
| ("1.000", q4["intervene_trial"]["with_trial"]["r_vs_trial"], "Qwen3-4B r_vs_trial"), | |
| ]: | |
| check(f"answers.md quotes {label} ({quoted})", float(quoted), float(actual), tol=6e-3) | |
| if quoted not in ans: | |
| failures.append(f"answers.md does not literally contain {quoted}") | |
| # ranges quoted in the README and answers | |
| rc = [m["intervene"]["all"]["r_vs_corr"] for m in models.values()] | |
| rt = [m["intervene"]["all"]["r_vs_true"] for m in models.values()] | |
| # the README quotes a range for the four correlation-tracking models; the fifth | |
| # (Qwen3-0.6B) is a degenerate-output case and must be disclosed, not averaged in | |
| four = [v for k, v in zip(models.keys(), rc) if k != "Qwen3-0.6B"] | |
| check("readme range corr(est,|corr|) low (4 models)", min(four), 0.888, tol=6e-3) | |
| check("readme range corr(est,|corr|) high (4 models)", max(four), 0.998, tol=6e-3) | |
| check("Qwen3-0.6B r_vs_corr (outlier)", dict(zip(models.keys(), rc))["Qwen3-0.6B"], 0.334, tol=6e-3) | |
| check("readme range corr(est,true) low", min(rt), 0.15, tol=0.006) | |
| check("readme range corr(est,true) high", max(rt), 0.28, tol=0.006) | |
| for must in ["0.888-0.998", "0.152-0.278", "Qwen3-0.6B"]: | |
| if must not in txt: | |
| failures.append(f"README does not disclose: {must}") | |
| for must in ["0.888-0.998", "0.152-0.278", "Qwen3-0.6B"]: | |
| if must not in ans: | |
| failures.append(f"answers.md does not disclose: {must}") | |
| check("readme range corr(est,true) low", min(rt), 0.15, tol=0.006) | |
| check("readme range corr(est,true) high", max(rt), 0.28, tol=0.006) | |
| # baselines quoted in the README/answers | |
| from baselines import corr_baseline, partial_corr_baseline, score_selection | |
| insts = {} | |
| for seed, spec in specs.items(): | |
| zsc, corr, do, _ = simulate(spec) | |
| insts[seed] = (zsc, corr, do) | |
| sel_p, sel_r, sel_f = [], [], [] | |
| pc_p, pc_r, pc_f = [], [], [] | |
| for path in sorted(glob.glob(os.path.join(RESULTS, "raw_*.json"))): | |
| blob = json.load(open(path, encoding="utf8")) | |
| for row in blob["results"]["select"]: | |
| zsc, corr, do = insts[row["instance"]] | |
| df = np.column_stack([zsc, np.zeros((zsc.shape[0], 1))]) | |
| # rebuild the two baselines with the independent correlation vector | |
| corr_scores = {v: corr[v] for v in CANDIDATE_NAMES} | |
| csel = sorted(corr_scores, key=lambda n: -abs(corr_scores[n]))[:3] | |
| # partial correlation, recomputed from the independent data | |
| X = zsc[:, : len(CANDIDATE_NAMES)] | |
| y = zsc[:, -1] | |
| R = np.corrcoef(np.column_stack([X, y]).T) | |
| P = np.linalg.pinv(R) | |
| ps = {} | |
| for i, v in enumerate(CANDIDATE_NAMES): | |
| dd = np.sqrt(P[i, i] * P[-1, -1]) | |
| ps[v] = float(-P[i, -1] / dd) if dd > 1e-12 else 0.0 | |
| psel = sorted(ps, key=lambda n: -abs(ps[n]))[:3] | |
| for sel, acc in ((csel, (sel_p, sel_r, sel_f)), (psel, (pc_p, pc_r, pc_f))): | |
| s = score_selection(sel, TRUE_PARENTS) | |
| acc[0].append(s["precision"]) | |
| acc[1].append(s["recall"]) | |
| acc[2].append(s["f1"]) | |
| check("corr baseline F1", float(np.mean(sel_f)), 0.571, tol=8e-3) | |
| check("partial-corr baseline F1", float(np.mean(pc_f)), 0.857, tol=8e-3) | |
| # ---- report ------------------------------------------------------------------- | |
| print(f"checks run: {checks}") | |
| if failures: | |
| print(f"FAILURES: {len(failures)}") | |
| for f in failures[:40]: | |
| print(" -", f) | |
| sys.exit(1) | |
| print("ALL CHECKS PASSED") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 20.6 kB
- Xet hash:
- 192b7c222735b9c0f561d9154072c5ac4f1b670940bbd26d6bb9c03ebd073394
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.