"""Full training pipeline: convert -> tokenizer (<=2h) -> prep -> generator (<=2h). python code/run_cmu.py --amc --work Each step logs to /runs/.log. /runs/STATUS.json = current step / state; /runs/ALERT.txt = something needs attention. Hang guard: a step whose log has not grown for --stall-min minutes is killed and reported. Resumable: steps listed in runs/STATUS.json as done are skipped (use --from to force a start step). Checkpoints are written to /ckpt (cmu_tok_v1_H.pt, cmu_gen_v1.pt). """ import os, sys, json, time, subprocess, argparse CODE = os.path.dirname(os.path.abspath(__file__)) REPO = os.path.dirname(CODE) PY = sys.executable W = RUNS = STEPS = None def setup(amc, work): global W, RUNS, STEPS W = os.path.abspath(work) RUNS = os.path.join(W, "runs") STEPS = [ ("convert", [os.path.join(CODE, "cmu_to272.py"), amc, os.path.join(REPO, "data", "cmu_files.csv"), os.path.join(W, "data", "motion_raw", "h272")]), ("tokenizer", [os.path.join(CODE, "tok_cmu.py"), "--seconds", "7200", "--tag", "cmu_tok_v1"]), ("prep", [os.path.join(CODE, "gen_prep.py")]), ("generator", [os.path.join(CODE, "gen_ar.py"), "--seconds", "7200", "--tag", "cmu_gen_v1"]), ] def now(): return time.strftime("%Y-%m-%d %H:%M:%S") def status(**kw): p = os.path.join(RUNS, "STATUS.json") st = json.load(open(p, encoding="utf-8")) if os.path.exists(p) else {"done": []} st.update(kw, updated=now()) json.dump(st, open(p, "w", encoding="utf-8"), ensure_ascii=False, indent=1) return st def alert(msg): with open(os.path.join(RUNS, "ALERT.txt"), "a", encoding="utf-8") as f: f.write("[%s] %s\n" % (now(), msg)) def main(): ap = argparse.ArgumentParser() ap.add_argument("--from", dest="start", default=None) ap.add_argument("--stall-min", type=float, default=30) ap.add_argument("--amc", required=True, help="folder with the CMU .asf/.amc files (allasfamc.zip, extracted)") ap.add_argument("--work", required=True, help="work folder (data/, ckpt/, runs/ are created here)") a = ap.parse_args() setup(a.amc, a.work) os.makedirs(RUNS, exist_ok=True) os.makedirs(os.path.join(W, "ckpt"), exist_ok=True) env = dict(os.environ, PARTLOCK_DIR=W, PYTHONIOENCODING="utf-8", PYTHONUNBUFFERED="1") st = status(pid=os.getpid(), state="running") names = [n for n, _ in STEPS] first = names.index(a.start) if a.start else 0 for name, args in STEPS[first:]: if not a.start and name in st.get("done", []): continue log = os.path.join(RUNS, name + ".log") status(step=name, step_started=now(), state="running") with open(log, "w", encoding="utf-8") as lf: lf.write("[%s] start %s\n" % (now(), " ".join(args))) lf.flush() p = subprocess.Popen([PY] + args, stdout=lf, stderr=subprocess.STDOUT, env=env, cwd=CODE) last_size, last_change = 0, time.time() while p.poll() is None: time.sleep(30) sz = os.path.getsize(log) if sz != last_size: last_size, last_change = sz, time.time() elif time.time() - last_change > a.stall_min * 60: p.kill() alert("%s: no log output for %d min -> killed (hang)" % (name, a.stall_min)) break rc = p.wait() if rc != 0: alert("%s failed (exit %s). see runs/%s.log" % (name, rc, name)) status(state="failed", failed_step=name, exit_code=rc) return st = status(done=st.get("done", []) + [name], last_done=name) status(state="all done", step=None) if __name__ == "__main__": main()