File size: 1,248 Bytes
f927ea3 a0912d2 f927ea3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 | import os,subprocess,json,time,sys
from pathlib import Path
root=Path('/data/nyako-training-150'); art=root/'artifacts'
env=os.environ.copy();env['PYTHONUNBUFFERED']='1';env['TOKENIZERS_PARALLELISM']='false';env['OMP_NUM_THREADS']='8'
cmd=[str(root/'venv/bin/python'),'scripts/dev/anima_train_network.py','--config_file',str(art/'config/train.toml')]
(art/'config/launch.json').write_text(json.dumps({'command':cmd,'cwd':str(root/'lora-scripts-next')},indent=2))
(art/'config/trainer_commit.txt').write_bytes(subprocess.check_output(['git','rev-parse','HEAD'],cwd=root/'lora-scripts-next'))
with (art/'config/packages.txt').open('w') as f:subprocess.run([str(root/'venv/bin/pip'),'freeze'],stdout=f)
(root/'status.json').write_text(json.dumps({'stage':'training','started':time.time()}))
with (art/'logs/train.log').open('w') as f:
result=subprocess.run(cmd,cwd=root/'lora-scripts-next',env=env,stdout=f,stderr=subprocess.STDOUT)
(root/'status.json').write_text(json.dumps({'stage':'training_finished' if result.returncode==0 else 'training_failed','returncode':result.returncode,'time':time.time()}))
if result.returncode: sys.exit(result.returncode)
subprocess.run([str(root/'venv/bin/python'),str(root/'scripts/upload.py')],check=True,env=env)
|