Buckets:
| """Audit without GPUs; explicit execution only after the dedicated cards are free.""" | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| import shlex | |
| import subprocess | |
| import sys | |
| from examples.gpu_mode.local_judge import DEFAULT_IMAGE, PREFIX, evaluate_submission, inspect_idle_gpu, parse_gpus | |
| from examples.gpu_mode.official import OFFICIAL | |
| from ttt_discover.runtime.topology import FOUR_GPU_PROFILE, validate_topology | |
| ROOT = Path(__file__).resolve().parents[2] | |
| def verify_image(image=DEFAULT_IMAGE): | |
| image_id = subprocess.check_output(["docker", "image", "inspect", image, "--format", "{{.Id}}"], text=True).strip() | |
| # Explicit runc + no NVIDIA devices: this import audit cannot allocate GPUs. | |
| proc = subprocess.run(["docker", "run", "--rm", "--runtime", "runc", "--network", "none", | |
| "-e", "NVIDIA_VISIBLE_DEVICES=void", "-e", "CUDA_VISIBLE_DEVICES=", image_id, "--audit"], | |
| text=True, capture_output=True, timeout=180) | |
| lines = [line[len(PREFIX):] for line in proc.stdout.splitlines() if line.startswith(PREFIX)] | |
| if proc.returncode or len(lines) != 1: | |
| raise RuntimeError(f"CPU image audit failed: {proc.stdout[-1500:]} {proc.stderr[-1500:]}") | |
| report = json.loads(lines[0]) | |
| if report.get("status") != "cpu_environment_verified": | |
| raise RuntimeError(f"Image is not aligned: {report}") | |
| report["image_id"] = image_id | |
| return report | |
| def verify_gpu_split(config, *, require_idle=False): | |
| model = set(validate_topology(config)) | |
| judges = parse_gpus(config["trimul_judge_gpus"]) | |
| if model & set(judges): | |
| raise ValueError("Kernel timing GPU overlaps LLM GPUs") | |
| if require_idle: | |
| for gpu in judges: | |
| inspect_idle_gpu(gpu) | |
| return {"model_gpus": sorted(model), "judge_gpus": judges} | |
| def require_calibration(config): | |
| path = ROOT.parent / "control/gpu_calibration.json" | |
| if not path.is_file(): | |
| raise RuntimeError("Run prepare_trimul.sh calibrate --execute on FREE judge GPUs before training") | |
| report = json.loads(path.read_text()) | |
| if report["audit"]["image_id"] != config["trimul_image_id"]: | |
| raise RuntimeError("Calibration used a different judge image") | |
| judges = set(parse_gpus(config["trimul_judge_gpus"])) | |
| rows = report.get("runs", []) | |
| if not rows or any(row.get("correctness") != 1 for row in rows): | |
| raise RuntimeError("GPU calibration did not pass the official verifier") | |
| if not judges <= {row["gpu_index"] for row in rows}: | |
| raise RuntimeError("Not all selected timing GPUs were calibrated") | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("action", choices=["audit", "calibrate", "step0", "baseline", "paired-baseline", "ours"]) | |
| parser.add_argument("--gpus", default="2,3") | |
| parser.add_argument("--image", default=DEFAULT_IMAGE) | |
| parser.add_argument("--updates", type=int, default=50) | |
| parser.add_argument("--repeats", type=int, default=3) | |
| parser.add_argument("--model-path", type=Path, default=Path("/mnt/localssd/models/gpt-oss-120b")) | |
| parser.add_argument("--shared-step0", type=Path, default=ROOT.parent / "shared/trimul_step0.json") | |
| parser.add_argument("--run-dir", type=Path) | |
| parser.add_argument("--output", type=Path) | |
| parser.add_argument("--execute", action="store_true") | |
| args = parser.parse_args() | |
| if args.action == "audit": | |
| report = verify_image(args.image) | |
| report["packages_freeze"] = subprocess.check_output([ | |
| "docker", "run", "--rm", "--runtime", "runc", "--network", "none", | |
| "-e", "NVIDIA_VISIBLE_DEVICES=void", "-e", "CUDA_VISIBLE_DEVICES=", | |
| "--entrypoint", "/usr/local/bin/uv", report["image_id"], | |
| "pip", "freeze", "--python", "/opt/venv/bin/python"], text=True) | |
| print(json.dumps(report, indent=2)) | |
| if args.output: | |
| args.output.parent.mkdir(parents=True, exist_ok=True) | |
| args.output.write_text(json.dumps(report, indent=2) + "\n") | |
| args.output.with_name("judge-packages.lock.txt").write_text(report["packages_freeze"]) | |
| return | |
| if args.action == "calibrate": | |
| plan = {"mode": "official-leaderboard", "kernel": "published kernel, calibration ONLY; never an ours-only root", | |
| "gpus": parse_gpus(args.gpus), "repeats": args.repeats, | |
| "execute": args.execute, "tests": 18, "benchmarks": 7} | |
| print(json.dumps(plan, indent=2), flush=True) | |
| if not args.execute: | |
| return | |
| report = verify_image(args.image) | |
| for gpu in parse_gpus(args.gpus): | |
| inspect_idle_gpu(gpu) | |
| code = (OFFICIAL / "results/kernel-engineering/trimul.py").read_text() | |
| runs = [evaluate_submission(code, gpus=str(gpu), image=report["image_id"], work_root=ROOT.parent / "calibration") | |
| for gpu in parse_gpus(args.gpus) for _ in range(args.repeats)] | |
| if not all(row["correctness"] == 1 for row in runs): | |
| raise RuntimeError("Published calibration kernel failed the official verifier") | |
| output = args.output or ROOT.parent / "control/gpu_calibration.json" | |
| output.parent.mkdir(parents=True, exist_ok=True) | |
| output.write_text(json.dumps({"audit": report, "runs": runs}, indent=2, default=str) + "\n") | |
| print(json.dumps({"gpu_execution_verified": True, "scores_us": [row["score_us"] for row in runs], "output": str(output)})) | |
| return | |
| run_dir = args.run_dir or ROOT.parent / "runs" / args.action | |
| command = ["bash", str(ROOT / "scripts/run.sh"), "trimul", "--method", | |
| "single_anchor" if args.action == "ours" else "baseline", "--model-path", str(args.model_path), | |
| "--run-dir", str(run_dir), "--execution-profile", FOUR_GPU_PROFILE, | |
| "--trimul-judge-gpus", args.gpus, "--updates", str(args.updates)] | |
| if args.action == "step0": | |
| command += ["--step0-only", "--step0-output", str(args.shared_step0)] | |
| if args.action in {"ours", "paired-baseline"}: | |
| command += ["--shared-step0", str(args.shared_step0)] | |
| print(shlex.join(command), flush=True) | |
| if args.execute: | |
| subprocess.run(command, cwd=ROOT, check=True) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 6.29 kB
- Xet hash:
- 308f23b1e34dd5fc4f620bef57a4bb98d6a8d6d7ac94d2cbe31bd1d3b55118ab
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.