PartLock-CMU / code /run_cmu.py
Yakimask's picture
Upload 31 files
c7ae99a verified
Raw History Blame Contribute Delete
3.87 kB
"""Full training pipeline: convert -> tokenizer (<=2h) -> prep -> generator (<=2h).
python code/run_cmu.py --amc <folder with the CMU .asf/.amc files> --work <empty work folder>
Each step logs to <work>/runs/<step>.log.
<work>/runs/STATUS.json = current step / state; <work>/runs/ALERT.txt = something needs attention.
Hang guard: a step whose log has not grown for --stall-min minutes is killed and reported.
Resumable: steps listed in runs/STATUS.json as done are skipped (use --from to force a start step).
Checkpoints are written to <work>/ckpt (cmu_tok_v1_H.pt, cmu_gen_v1.pt).
"""
import os, sys, json, time, subprocess, argparse
CODE = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.dirname(CODE)
PY = sys.executable
W = RUNS = STEPS = None
def setup(amc, work):
global W, RUNS, STEPS
W = os.path.abspath(work)
RUNS = os.path.join(W, "runs")
STEPS = [
("convert", [os.path.join(CODE, "cmu_to272.py"), amc, os.path.join(REPO, "data", "cmu_files.csv"),
os.path.join(W, "data", "motion_raw", "h272")]),
("tokenizer", [os.path.join(CODE, "tok_cmu.py"), "--seconds", "7200", "--tag", "cmu_tok_v1"]),
("prep", [os.path.join(CODE, "gen_prep.py")]),
("generator", [os.path.join(CODE, "gen_ar.py"), "--seconds", "7200", "--tag", "cmu_gen_v1"]),
]
def now():
return time.strftime("%Y-%m-%d %H:%M:%S")
def status(**kw):
p = os.path.join(RUNS, "STATUS.json")
st = json.load(open(p, encoding="utf-8")) if os.path.exists(p) else {"done": []}
st.update(kw, updated=now())
json.dump(st, open(p, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
return st
def alert(msg):
with open(os.path.join(RUNS, "ALERT.txt"), "a", encoding="utf-8") as f:
f.write("[%s] %s\n" % (now(), msg))
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--from", dest="start", default=None)
ap.add_argument("--stall-min", type=float, default=30)
ap.add_argument("--amc", required=True, help="folder with the CMU .asf/.amc files (allasfamc.zip, extracted)")
ap.add_argument("--work", required=True, help="work folder (data/, ckpt/, runs/ are created here)")
a = ap.parse_args()
setup(a.amc, a.work)
os.makedirs(RUNS, exist_ok=True)
os.makedirs(os.path.join(W, "ckpt"), exist_ok=True)
env = dict(os.environ, PARTLOCK_DIR=W, PYTHONIOENCODING="utf-8", PYTHONUNBUFFERED="1")
st = status(pid=os.getpid(), state="running")
names = [n for n, _ in STEPS]
first = names.index(a.start) if a.start else 0
for name, args in STEPS[first:]:
if not a.start and name in st.get("done", []):
continue
log = os.path.join(RUNS, name + ".log")
status(step=name, step_started=now(), state="running")
with open(log, "w", encoding="utf-8") as lf:
lf.write("[%s] start %s\n" % (now(), " ".join(args)))
lf.flush()
p = subprocess.Popen([PY] + args, stdout=lf, stderr=subprocess.STDOUT, env=env, cwd=CODE)
last_size, last_change = 0, time.time()
while p.poll() is None:
time.sleep(30)
sz = os.path.getsize(log)
if sz != last_size:
last_size, last_change = sz, time.time()
elif time.time() - last_change > a.stall_min * 60:
p.kill()
alert("%s: no log output for %d min -> killed (hang)" % (name, a.stall_min))
break
rc = p.wait()
if rc != 0:
alert("%s failed (exit %s). see runs/%s.log" % (name, rc, name))
status(state="failed", failed_step=name, exit_code=rc)
return
st = status(done=st.get("done", []) + [name], last_done=name)
status(state="all done", step=None)
if __name__ == "__main__":
main()