| """Serialize fresh subprocesses and record GPU state across process boundaries.""" |
| import json |
| import subprocess |
| import sys |
| import time |
| from pathlib import Path |
|
|
|
|
| def main(): |
| root = Path('artifacts/eval-corrected') |
| root.mkdir(exist_ok=False) |
| jobs = [] |
| for precision in ('bf16', 'int8'): |
| jobs.append((precision, ['-m', 'scripts.benchmark', '--model', 'models/'+precision, |
| '--output', str(root/precision)])) |
| for precision in ('bf16', 'int8'): |
| jobs.append(('young_'+precision, ['-m', 'scripts.benchmark', '--model', 'models/'+precision, |
| '--output', str(root/'young_woman'/precision), '--cases', 'benchmarks/young_woman.json'])) |
| for name, command in jobs: |
| before = subprocess.check_output(['nvidia-smi'], text=True) |
| with (root/(name+'.log')).open('w', encoding='utf-8') as log: |
| proc = subprocess.Popen([sys.executable, *command], stdout=log, stderr=subprocess.STDOUT) |
| print('Started', name, 'PID', proc.pid, flush=True) |
| code = proc.wait() |
| after = subprocess.check_output(['nvidia-smi'], text=True) |
| output = Path(command[command.index('--output')+1]) |
| env_path = output/'environment.json' |
| inference_pid = json.loads(env_path.read_text())['pid'] if env_path.exists() else None |
| row = dict(name=name, pid=proc.pid, returncode=code, command=command, |
| inference_pid=inference_pid, gpu_before=before, gpu_after=after, completed=time.time()) |
| with (root/'process-isolation.jsonl').open('a', encoding='utf-8') as f: |
| f.write(json.dumps(row)+'\n') |
| print('Completed', name, 'exit', code, flush=True) |
| if code: |
| raise RuntimeError('Benchmark failed: '+name) |
|
|
|
|
| if __name__ == '__main__': |
| main() |
|
|
