File size: 1,724 Bytes
35730be
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
#!/usr/bin/env python3
"""Launch one owned finite NPU task; its process/log/control remain visible."""
from pathlib import Path
import argparse
import json
import os
import subprocess
import sys
import time
p = argparse.ArgumentParser()
p.add_argument('--root', type=Path, required=True)
p.add_argument('--name', required=True)
p.add_argument('--module', choices=['trainer', 'prepare'], required=True)
p.add_argument('--distributed', action='store_true')
p.add_argument('--port', type=int, default=29582)
p.add_argument('arguments', nargs=argparse.REMAINDER)
a = p.parse_args()
r = a.root.resolve()
if any(x not in 'abcdefghijklmnopqrstuvwxyz0123456789-_' for x in a.name):
    raise ValueError('Safe process name required')
args = a.arguments[1:] if a.arguments[:1] == ['--'] else a.arguments
cmd = [sys.executable, '-I']
if a.distributed:
    cmd += ['-m', 'torch.distributed.run', '--nproc_per_node=8', '--master_port='+str(a.port)]
cmd += [str(r/'scripts/entry.py'), a.module, '--root', str(r), '--source-dir', str(r/'trusted/src'), *args]
env = dict(os.environ, LD_PRELOAD='/lib/aarch64-linux-gnu/libgomp.so.1',
           OMP_NUM_THREADS='2' if a.distributed else '8', HCCL_EXEC_TIMEOUT='1200',
           TMPDIR=str(r/'tmp'), TOKENIZERS_PARALLELISM='false', HF_HOME=str(r/'hf-cache'))
logfile = r/'logs'/f'{a.name}.log'
with logfile.open('ab', buffering=0) as log:
    proc = subprocess.Popen(cmd, stdout=log, stderr=subprocess.STDOUT,
                            start_new_session=True, cwd='/mnt/workspace', env=env)
info = {'pid':proc.pid,'command':cmd,'log':str(logfile.relative_to(r)),'started_at':time.time()}
(r/'control'/f'{a.name}_process.json').write_text(json.dumps(info,indent=2))
print(json.dumps(info))