timestral-coreai / scripts /run_guarded.py
jbilcke's picture
Publish Timestral CoreAI models, native Swift SDK and validated Python runtime
506f40d verified
Raw History Blame Contribute Delete
3.65 kB
"""Run one local validation process with a time and memory budget."""
import argparse
import json
import os
from pathlib import Path
import signal
import subprocess
import time
import psutil
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--timeout', type=float, default=120)
parser.add_argument('--min-free-gib', type=float, default=6)
parser.add_argument('--max-rss-gib', type=float, default=4)
parser.add_argument('--report', type=Path, required=True)
parser.add_argument('command', nargs=argparse.REMAINDER)
args = parser.parse_args()
command = args.command[1:] if args.command[:1] == ['--'] else args.command
if not command or min(args.timeout, args.min_free_gib, args.max_rss_gib) <= 0:
parser.error('provide positive budgets and a command after --')
report = dict(command=command, timeout_seconds=args.timeout,
minimum_free_gib=args.min_free_gib, maximum_process_rss_gib=args.max_rss_gib,
max_observed_rss_gib=0, min_observed_free_gib=psutil.virtual_memory().available / 2**30,
completed=False, stopped_reason=None)
started = time.monotonic()
process = None
try:
if report['min_observed_free_gib'] < args.min_free_gib:
report['stopped_reason'] = 'insufficient free memory before launch'
return 125
process = subprocess.Popen(command, start_new_session=True)
parent = psutil.Process(process.pid)
while process.poll() is None:
free = psutil.virtual_memory().available / 2**30
rss = 0
try:
for child in [parent] + parent.children(recursive=True):
try:
rss += child.memory_info().rss / 2**30
except psutil.NoSuchProcess:
pass
except psutil.NoSuchProcess:
pass
report['min_observed_free_gib'] = min(free, report['min_observed_free_gib'])
report['max_observed_rss_gib'] = max(rss, report['max_observed_rss_gib'])
if free < args.min_free_gib:
report['stopped_reason'] = 'free memory cutoff'
break
if rss > args.max_rss_gib:
report['stopped_reason'] = 'process memory cutoff'
break
if time.monotonic() - started > args.timeout:
report['stopped_reason'] = 'time limit'
break
time.sleep(.2)
if report['stopped_reason'] is None:
report['exit_code'] = process.wait()
report['completed'] = True
return report['exit_code']
return 124
except (KeyboardInterrupt, Exception) as exc:
report['stopped_reason'] = f'{type(exc).__name__}: {exc}'
return 130
finally:
if process is not None and process.poll() is None:
try:
os.killpg(process.pid, signal.SIGTERM)
except ProcessLookupError:
pass
try:
process.wait(timeout=3)
except subprocess.TimeoutExpired:
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
process.wait(timeout=3)
report['elapsed_seconds'] = time.monotonic() - started
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(json.dumps(report, indent=2) + '\n')
print(json.dumps(report), flush=True)
if __name__ == '__main__':
raise SystemExit(main())