MARICL / examples /synthetic_data /regression.py
rezapop's picture
Add MARICL paper (PDF + LaTeX source) and code (part 3)
e4a79f7 verified
Raw History Blame Contribute Delete
9.45 kB
"""Regression example on synthetic data: Claude fixes a linear model's residuals.
y = 0.4 x1 - 0.3 x3 + 2 sigmoid(3 x2 x5) + 1.2 x4^2 / (1 + x4^2) + eps,
x ~ N(0, I_6), eps ~ N(0, 0.1^2), N = 1000 (600 / 200 / 200), x6 irrelevant.
Protocol (paper Sec. 3): residuals, high-residual pool, critiques and p_k on train; the retained
iterate m_k* is chosen on validation; test is used only for final metrics. Predictions are
reported for the gated headline configuration (Eq. 21) and the ungated variant (Eq. 1).
Usage:
python regression.py --seed 0 # real run (needs Anthropic credentials)
python regression.py --seed 0 --mock --T 3 # offline pipeline check, no API calls
"""
from __future__ import annotations
import argparse
import ast
import json
import os
import random
import sys
import time
import numpy as np
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_absolute_error, r2_score
from xgboost import XGBRegressor
from claude_backend import ClaudeLLM, install_import_shims
install_import_shims()
import maricl_core as mc # noqa: E402 (released MARICL implementation, unmodified)
from maricl_headline import maricl_predict, select # noqa: E402
FEATURES = [f"x{i}" for i in range(1, 7)]
FEATURE_DESCS = ["standardised input feature (semantics unknown)"] * len(FEATURES)
DOMAIN = ("Synthetic regression benchmark with standardised inputs. Feature semantics are "
"unknown and no domain knowledge applies.")
N, D = 1000, 6
def sigmoid(z):
return 1.0 / (1.0 + np.exp(-np.clip(z, -50, 50)))
def true_function(X):
x1, x2, x3, x4, x5 = (X[:, i] for i in range(5))
return 0.4 * x1 - 0.3 * x3 + 2.0 * sigmoid(3.0 * x2 * x5) + 1.2 * x4**2 / (1.0 + x4**2)
def make_data(seed):
rng = np.random.default_rng(seed)
X = rng.standard_normal((N, D))
f = true_function(X)
y = f + rng.normal(0.0, 0.1, N)
lo, hi = float(y.min()), float(y.max())
idx = np.random.RandomState(seed).permutation(N)
split = {"train": idx[:600], "val": idx[600:800], "test": idx[800:]}
return X, (y - lo) / (hi - lo), (f - lo) / (hi - lo), split, (lo, hi)
# ---------------------------------------------------------------- recovery (syntax tree)
def _names(node):
return {n.id for n in ast.walk(node) if isinstance(n, ast.Name)}
def term_flags(expr: str) -> dict:
tree = ast.parse(expr, mode="eval")
inter = spurious12 = sat4 = False
for n in ast.walk(tree):
if isinstance(n, ast.BinOp) and isinstance(n.op, ast.Mult):
l, r = _names(n.left), _names(n.right)
if ("x2" in l and "x5" in r) or ("x5" in l and "x2" in r):
inter = True
if ("x1" in l and "x2" in r) or ("x2" in l and "x1" in r):
spurious12 = True
if "x4" in l and "x4" in r:
sat4 = True
if isinstance(n, ast.BinOp) and isinstance(n.op, ast.Pow) and "x4" in _names(n.left):
sat4 = True
if isinstance(n, ast.BinOp) and isinstance(n.op, ast.Div) and "x4" in _names(n.right):
sat4 = True
if isinstance(n, ast.Call) and isinstance(n.func, ast.Name) and n.func.id != "clip" \
and "x4" in _names(n):
sat4 = True
used = sorted(_names(tree) & set(FEATURES))
return {"interaction_x2x5": inter, "nonlinear_x4": sat4, "uses_x6": "x6" in used,
"spurious_x1x2": spurious12, "features_used": used}
# ---------------------------------------------------------------- offline stand-in
class _MockRunnable:
def __init__(self, schema, rng):
self.schema, self.rng = schema, rng
def invoke(self, messages):
name = self.schema.__name__
if name == "Hypothesis":
return self.schema(hypothesised_pattern="mock", implicated_features=["x2", "x5", "x4"],
functional_form_guess="a*sigmoid(b*x2*x5) + c*x4**2/(1+x4**2)",
rationale="offline mock")
if name == "Correction":
a, b, c = self.rng.uniform(0.05, 0.3), self.rng.uniform(0.0, 0.15), self.rng.uniform(0, 0.2)
return self.schema(T_k="mock", f_k=f"{a:.3f}*sigmoid(3*x2*x5) + {b:.3f}*x4**2/(1+x4**2) - {c:.3f}")
return self.schema(diagnosis="mock", structural_or_coefficient="coefficient", suggested_change="mock")
class MockLLM:
def __init__(self, seed):
self.rng = random.Random(seed)
def with_structured_output(self, schema):
return _MockRunnable(schema, self.rng)
# ---------------------------------------------------------------- main
def load_dotenv(path=".env"):
"""Read KEY=VALUE lines into the environment (existing variables win). Values are never printed."""
if not os.path.exists(path):
return
for line in open(path):
line = line.strip()
if line and not line.startswith("#") and "=" in line:
key, value = line.split("=", 1)
os.environ.setdefault(key.strip(), value.strip().strip('"').strip("'"))
def run_experiment(seed, T, llm, llm_creative, out_dir, tag, calls_path=None, fit_log=None):
"""Run one seed end to end; returns the results dict (also written to out_dir/tag.json)."""
import contextlib
X, ys, fs, sp, (lo, hi) = make_data(seed)
tr, va, te = sp["train"], sp["val"], sp["test"]
lin = LinearRegression().fit(X[tr], ys[tr])
b_tr, b_va, b_te = lin.predict(X[tr]), lin.predict(X[va]), lin.predict(X[te])
mc.configure_llms(llm, llm_creative)
model = mc.MARICL(task="regression", K=2, kappa=0.3, T=T, B=10, p_min=0.1,
gamma=2.0, tau_fail=0.1, tau_reg=0.2, verbose=True)
t0 = time.time()
sink = open(fit_log, "w") if fit_log else None
try:
with contextlib.redirect_stdout(sink) if sink else contextlib.nullcontext():
model.fit(X[tr], ys[tr], base_model=lin, feature_names=FEATURES,
feature_descs=FEATURE_DESCS, domain_context=DOMAIN, reg_target_range=(0.0, 1.0))
finally:
if sink:
sink.close()
fit_seconds = time.time() - t0
xgb = XGBRegressor(random_state=seed).fit(X[tr], ys[tr])
def metrics(pred, idx):
return {"r2": float(r2_score(ys[idx], pred)), "mae": float(mean_absolute_error(ys[idx], pred))}
chosen, scores, ts = select(model, X[tr], ys[tr], X[va], ys[va], b_tr, b_va, T)
res = {
"seed": seed, "T": T, "target_range": [lo, hi], "fit_seconds": round(fit_seconds, 1),
"linear": metrics(b_te, te), "xgboost": metrics(xgb.predict(X[te]), te),
"oracle": metrics(fs[te], te),
"maricl_gated": metrics(maricl_predict(model, X[te], b_te, chosen, scores, True), te),
"maricl_ungated": metrics(maricl_predict(model, X[te], b_te, chosen, scores, False), te),
"corrections": [], "budget_curve": [],
}
for k, (s, c, p, t) in enumerate(zip(model.states_, chosen, scores, ts)):
res["corrections"].append({
"k": k + 1, "p_k": p, "retained_t": t,
"hypothesis": s.hypothesis.hypothesised_pattern,
"form_guess": s.hypothesis.functional_form_guess,
"T_k": c.T_k if c else None, "f_k": c.f_k if c else None,
"flags": term_flags(c.f_k) if c else None,
"history": [{"t": e["t"], "f_k": e["correction"].f_k, "train_mae": e["loss"]}
for e in s.history],
})
for t_max in range(T + 1):
ch, sc, _ = select(model, X[tr], ys[tr], X[va], ys[va], b_tr, b_va, t_max)
res["budget_curve"].append({
"T": t_max,
"gated": metrics(maricl_predict(model, X[te], b_te, ch, sc, True), te)["r2"],
"ungated": metrics(maricl_predict(model, X[te], b_te, ch, sc, False), te)["r2"],
})
if calls_path and os.path.exists(calls_path):
calls = [json.loads(line) for line in open(calls_path)]
res["calls"] = {"n": len(calls), "input_tokens": sum(c["input_tokens"] for c in calls),
"output_tokens": sum(c["output_tokens"] for c in calls)}
os.makedirs(out_dir, exist_ok=True)
with open(os.path.join(out_dir, f"{tag}.json"), "w") as fh:
json.dump(res, fh, indent=2)
return res
def main():
load_dotenv(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".env"))
ap = argparse.ArgumentParser()
ap.add_argument("--seed", type=int, required=True)
ap.add_argument("--T", type=int, default=10)
ap.add_argument("--model", default="claude-opus-5-5")
ap.add_argument("--effort", default="medium")
ap.add_argument("--mock", action="store_true")
ap.add_argument("--out", default="results_api/regression")
args = ap.parse_args()
os.makedirs(args.out, exist_ok=True)
tag = f"{'mock_' if args.mock else ''}seed{args.seed}"
calls_path = os.path.join(args.out, f"calls_{tag}.jsonl")
if os.path.exists(calls_path):
os.remove(calls_path)
if args.mock:
llm = llm_creative = MockLLM(args.seed)
else:
llm = ClaudeLLM(args.model, args.effort, calls_path, role="base")
llm_creative = ClaudeLLM(args.model, args.effort, calls_path, role="encoder")
res = run_experiment(args.seed, args.T, llm, llm_creative, args.out, tag, calls_path)
print(json.dumps({k: res[k] for k in ("linear", "xgboost", "maricl_gated", "maricl_ungated", "oracle")}))
if __name__ == "__main__":
sys.exit(main())