File size: 1,827 Bytes
ba48d54 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 | """Serialize fresh subprocesses and record GPU state across process boundaries."""
import json
import subprocess
import sys
import time
from pathlib import Path
def main():
root = Path('artifacts/eval-corrected')
root.mkdir(exist_ok=False)
jobs = []
for precision in ('bf16', 'int8'):
jobs.append((precision, ['-m', 'scripts.benchmark', '--model', 'models/'+precision,
'--output', str(root/precision)]))
for precision in ('bf16', 'int8'):
jobs.append(('young_'+precision, ['-m', 'scripts.benchmark', '--model', 'models/'+precision,
'--output', str(root/'young_woman'/precision), '--cases', 'benchmarks/young_woman.json']))
for name, command in jobs:
before = subprocess.check_output(['nvidia-smi'], text=True)
with (root/(name+'.log')).open('w', encoding='utf-8') as log:
proc = subprocess.Popen([sys.executable, *command], stdout=log, stderr=subprocess.STDOUT)
print('Started', name, 'PID', proc.pid, flush=True)
code = proc.wait()
after = subprocess.check_output(['nvidia-smi'], text=True)
output = Path(command[command.index('--output')+1])
env_path = output/'environment.json'
inference_pid = json.loads(env_path.read_text())['pid'] if env_path.exists() else None
row = dict(name=name, pid=proc.pid, returncode=code, command=command,
inference_pid=inference_pid, gpu_before=before, gpu_after=after, completed=time.time())
with (root/'process-isolation.jsonl').open('a', encoding='utf-8') as f:
f.write(json.dumps(row)+'\n')
print('Completed', name, 'exit', code, flush=True)
if code:
raise RuntimeError('Benchmark failed: '+name)
if __name__ == '__main__':
main()
|