zefancainlp's picture
download
raw
6.29 kB
"""Audit without GPUs; explicit execution only after the dedicated cards are free."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import shlex
import subprocess
import sys
from examples.gpu_mode.local_judge import DEFAULT_IMAGE, PREFIX, evaluate_submission, inspect_idle_gpu, parse_gpus
from examples.gpu_mode.official import OFFICIAL
from ttt_discover.runtime.topology import FOUR_GPU_PROFILE, validate_topology
ROOT = Path(__file__).resolve().parents[2]
def verify_image(image=DEFAULT_IMAGE):
image_id = subprocess.check_output(["docker", "image", "inspect", image, "--format", "{{.Id}}"], text=True).strip()
# Explicit runc + no NVIDIA devices: this import audit cannot allocate GPUs.
proc = subprocess.run(["docker", "run", "--rm", "--runtime", "runc", "--network", "none",
"-e", "NVIDIA_VISIBLE_DEVICES=void", "-e", "CUDA_VISIBLE_DEVICES=", image_id, "--audit"],
text=True, capture_output=True, timeout=180)
lines = [line[len(PREFIX):] for line in proc.stdout.splitlines() if line.startswith(PREFIX)]
if proc.returncode or len(lines) != 1:
raise RuntimeError(f"CPU image audit failed: {proc.stdout[-1500:]} {proc.stderr[-1500:]}")
report = json.loads(lines[0])
if report.get("status") != "cpu_environment_verified":
raise RuntimeError(f"Image is not aligned: {report}")
report["image_id"] = image_id
return report
def verify_gpu_split(config, *, require_idle=False):
model = set(validate_topology(config))
judges = parse_gpus(config["trimul_judge_gpus"])
if model & set(judges):
raise ValueError("Kernel timing GPU overlaps LLM GPUs")
if require_idle:
for gpu in judges:
inspect_idle_gpu(gpu)
return {"model_gpus": sorted(model), "judge_gpus": judges}
def require_calibration(config):
path = ROOT.parent / "control/gpu_calibration.json"
if not path.is_file():
raise RuntimeError("Run prepare_trimul.sh calibrate --execute on FREE judge GPUs before training")
report = json.loads(path.read_text())
if report["audit"]["image_id"] != config["trimul_image_id"]:
raise RuntimeError("Calibration used a different judge image")
judges = set(parse_gpus(config["trimul_judge_gpus"]))
rows = report.get("runs", [])
if not rows or any(row.get("correctness") != 1 for row in rows):
raise RuntimeError("GPU calibration did not pass the official verifier")
if not judges <= {row["gpu_index"] for row in rows}:
raise RuntimeError("Not all selected timing GPUs were calibrated")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("action", choices=["audit", "calibrate", "step0", "baseline", "paired-baseline", "ours"])
parser.add_argument("--gpus", default="2,3")
parser.add_argument("--image", default=DEFAULT_IMAGE)
parser.add_argument("--updates", type=int, default=50)
parser.add_argument("--repeats", type=int, default=3)
parser.add_argument("--model-path", type=Path, default=Path("/mnt/localssd/models/gpt-oss-120b"))
parser.add_argument("--shared-step0", type=Path, default=ROOT.parent / "shared/trimul_step0.json")
parser.add_argument("--run-dir", type=Path)
parser.add_argument("--output", type=Path)
parser.add_argument("--execute", action="store_true")
args = parser.parse_args()
if args.action == "audit":
report = verify_image(args.image)
report["packages_freeze"] = subprocess.check_output([
"docker", "run", "--rm", "--runtime", "runc", "--network", "none",
"-e", "NVIDIA_VISIBLE_DEVICES=void", "-e", "CUDA_VISIBLE_DEVICES=",
"--entrypoint", "/usr/local/bin/uv", report["image_id"],
"pip", "freeze", "--python", "/opt/venv/bin/python"], text=True)
print(json.dumps(report, indent=2))
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2) + "\n")
args.output.with_name("judge-packages.lock.txt").write_text(report["packages_freeze"])
return
if args.action == "calibrate":
plan = {"mode": "official-leaderboard", "kernel": "published kernel, calibration ONLY; never an ours-only root",
"gpus": parse_gpus(args.gpus), "repeats": args.repeats,
"execute": args.execute, "tests": 18, "benchmarks": 7}
print(json.dumps(plan, indent=2), flush=True)
if not args.execute:
return
report = verify_image(args.image)
for gpu in parse_gpus(args.gpus):
inspect_idle_gpu(gpu)
code = (OFFICIAL / "results/kernel-engineering/trimul.py").read_text()
runs = [evaluate_submission(code, gpus=str(gpu), image=report["image_id"], work_root=ROOT.parent / "calibration")
for gpu in parse_gpus(args.gpus) for _ in range(args.repeats)]
if not all(row["correctness"] == 1 for row in runs):
raise RuntimeError("Published calibration kernel failed the official verifier")
output = args.output or ROOT.parent / "control/gpu_calibration.json"
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps({"audit": report, "runs": runs}, indent=2, default=str) + "\n")
print(json.dumps({"gpu_execution_verified": True, "scores_us": [row["score_us"] for row in runs], "output": str(output)}))
return
run_dir = args.run_dir or ROOT.parent / "runs" / args.action
command = ["bash", str(ROOT / "scripts/run.sh"), "trimul", "--method",
"single_anchor" if args.action == "ours" else "baseline", "--model-path", str(args.model_path),
"--run-dir", str(run_dir), "--execution-profile", FOUR_GPU_PROFILE,
"--trimul-judge-gpus", args.gpus, "--updates", str(args.updates)]
if args.action == "step0":
command += ["--step0-only", "--step0-output", str(args.shared_step0)]
if args.action in {"ours", "paired-baseline"}:
command += ["--shared-step0", str(args.shared_step0)]
print(shlex.join(command), flush=True)
if args.execute:
subprocess.run(command, cwd=ROOT, check=True)
if __name__ == "__main__":
main()

Xet Storage Details

Size:
6.29 kB
·
Xet hash:
308f23b1e34dd5fc4f620bef57a4bb98d6a8d6d7ac94d2cbe31bd1d3b55118ab

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.