Upload folder using huggingface_hub

src/mathcompose/common/pyexec.py

@@ -0,0 +1,67 @@
+"""Sandboxed Python execution for the code-verifying verifier (GenPRM-style).
+
+The verifier can emit ```python ... ``` blocks to *check* arithmetic (the thing a
+1.5B model can't do by reading); we execute them and feed stdout back. Model-
+generated code is untrusted, so we run it in a subprocess with:
+  * a wall-clock timeout,
+  * no inherited environment / no network creds,
+  * a scratch cwd,
+  * output truncation.
+
+This is defense-in-depth, not a hard jail — run it only in a disposable
+environment (Colab/Modal VM), never on a privileged host.
+"""
+from __future__ import annotations
+
+import subprocess
+import sys
+import tempfile
+from dataclasses import dataclass
+
+
+@dataclass
+class ExecResult:
+    ok: bool
+    stdout: str
+    stderr: str
+
+    def as_feedback(self, max_chars: int = 600) -> str:
+        if self.ok:
+            out = self.stdout.strip() or "(no output)"
+            return out[:max_chars]
+        err = (self.stderr.strip() or "error").splitlines()[-1]
+        return f"[error] {err[:max_chars]}"
+
+
+_PREAMBLE = "import math, fractions, itertools\nfrom fractions import Fraction\n"
+
+
+def run_python(code: str, timeout: float = 5.0, max_output: int = 4000) -> ExecResult:
+    """Execute a short snippet in a subprocess and capture stdout/stderr."""
+    src = _PREAMBLE + code
+    try:
+        with tempfile.TemporaryDirectory() as tmp:
+            proc = subprocess.run(
+                [sys.executable, "-I", "-c", src],   # -I: isolated (ignore env/site)
+                capture_output=True, text=True, timeout=timeout, cwd=tmp,
+                env={"PYTHONHASHSEED": "0"},
+            )
+        return ExecResult(
+            ok=(proc.returncode == 0),
+            stdout=proc.stdout[:max_output],
+            stderr=proc.stderr[:max_output],
+        )
+    except subprocess.TimeoutExpired:
+        return ExecResult(ok=False, stdout="", stderr=f"timeout after {timeout}s")
+    except Exception as e:  # noqa: BLE001
+        return ExecResult(ok=False, stdout="", stderr=f"{type(e).__name__}: {e}")
+
+
+# Extract fenced python blocks from a model critique.
+import re  # noqa: E402
+
+_CODE_BLOCK = re.compile(r"```(?:python)?\s*\n(.*?)```", re.DOTALL)
+
+
+def extract_code_blocks(text: str) -> list[str]:
+    return [m.group(1).strip() for m in _CODE_BLOCK.finditer(text)]

src/mathcompose/eval/run.py

@@ -28,9 +28,40 @@ def _save(name: str, tag: str, payload: dict) -> None:
     print(json.dumps(payload, indent=2))
 
 
+DEFAULT_BASE = "Qwen/Qwen2.5-Math-1.5B-Instruct"
+
+
+def _base_from_adapter(adapter):
+    """Read base_model_name_or_path from an adapter's config (local dir or hub id)."""
+    import json
+    import os
+
+    try:
+        if os.path.isdir(adapter):
+            cfg = json.load(open(os.path.join(adapter, "adapter_config.json")))
+        else:
+            from huggingface_hub import hf_hub_download
+            cfg = json.load(open(hf_hub_download(adapter, "adapter_config.json")))
+        return cfg.get("base_model_name_or_path")
+    except Exception:
+        return None
+
+
+def _resolve_base(base_id, adapter):
+    """Explicit --base-id wins; else infer from the adapter; else the 1.5B default."""
+    if base_id:
+        return base_id
+    if adapter:
+        inferred = _base_from_adapter(adapter)
+        if inferred:
+            print(f"[eval] base inferred from adapter: {inferred}")
+            return inferred
+    return DEFAULT_BASE
+
+
 def _runner(base_id, adapter):
     from .runners import HFRunner
-    return HFRunner(base_id, adapter_path=adapter)
+    return HFRunner(_resolve_base(base_id, adapter), adapter_path=adapter)
 
 
 def _prompts_per_batch(a):
@@ -77,8 +108,8 @@ def cmd_compose(a):
     from .compose import evaluate_composition, make_v_scorer
     from .runners import HFRunner, make_solve_fn
 
-    gen = HFRunner(a.base_id, adapter_path=a.gen_adapter)
-    ver = HFRunner(a.base_id, adapter_path=a.ver_adapter)
+    gen = HFRunner(_resolve_base(a.base_id, a.gen_adapter), adapter_path=a.gen_adapter)
+    ver = HFRunner(_resolve_base(a.base_id, a.ver_adapter), adapter_path=a.ver_adapter)
     solve_fn = make_solve_fn(gen, n=a.n, temperature=a.temperature)
     v_score_fn = make_v_scorer(ver, k=a.ver_k)
 
@@ -107,7 +138,8 @@ def cmd_report(a):
 
 def main() -> None:
     ap = argparse.ArgumentParser(description=__doc__)
-    ap.add_argument("--base-id", default="Qwen/Qwen2.5-Math-1.5B-Instruct")
+    ap.add_argument("--base-id", default=None,
+                    help="base model; if omitted, inferred from the adapter (else 1.5B default)")
     sub = ap.add_subparsers(dest="cmd", required=True)
 
     p = sub.add_parser("processbench")

tests/test_pyexec.py

@@ -0,0 +1,24 @@
+"""Sandboxed executor for the code-verifying verifier (M1c)."""
+from mathcompose.common.pyexec import run_python, extract_code_blocks
+
+
+def test_arithmetic_and_output():
+    assert run_python("print(sum(range(101)))").as_feedback() == "5050"
+    assert run_python("print(6*7)").stdout.strip() == "42"
+
+
+def test_timeout_is_caught():
+    r = run_python("while True: pass", timeout=1.0)
+    assert not r.ok and "timeout" in r.stderr
+
+
+def test_error_is_caught():
+    r = run_python("print(1/0)")
+    assert not r.ok and "ZeroDivisionError" in r.as_feedback()
+
+
+def test_extract_code_blocks():
+    text = "a\n```python\nprint(1)\n```\nb\n```\nprint(2)\n```"
+    blocks = extract_code_blocks(text)
+    assert blocks == ["print(1)", "print(2)"]
+    assert run_python(blocks[0]).as_feedback() == "1"