Download bin/ec from codemaivanngu/job-manager: direct link, hf CLI and curl.
- Browser
- Download file 14.5 kB
-
https://huggingface.co/codemaivanngu/job-manager/resolve/main/bin/ec
- Command line
-
hf download hf://codemaivanngu/job-manager/bin/ec
-
curl -L -o ec https://huggingface.co/codemaivanngu/job-manager/resolve/main/bin/ec
14.5 kB
| #!/usr/bin/env python3 | |
| """eval-core CLI (v0.2). File-based, multi-machine, per-checkpoint. | |
| ec host --gpus 0,1,3 | |
| ec add --case <dir> --run R --steps 40..312 --bench all --seeds 42,43,44 | |
| ec status | ec why <job> | ec beat <job> | ec gc --older-than 20m | |
| ec bank ls|import <case>/eval|verify | |
| """ | |
| import argparse, datetime, hashlib, json, os, pathlib, socket, time | |
| ROOT = pathlib.Path(os.environ.get("EC_ROOT", "/workspace/storage-shared/nlp/tungks/eval-core")) | |
| HOST = os.environ.get("EC_HOST") or socket.gethostname().split(".")[0] | |
| Q, BANK, LOCKS, RUN = ROOT/"queue", ROOT/"bank", ROOT/"locks", ROOT/"run" | |
| HOSTS, BEAT = ROOT/"hosts", ROOT/"heartbeats" | |
| STOP = ROOT/"stops" | |
| BENCHES = ["gsm8k", "math500", "mbpp", "live-code-bench-v6"] | |
| def now(): return datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds") | |
| def jload(p, d=None): | |
| try: return json.loads(pathlib.Path(p).read_text()) | |
| except Exception: return d | |
| def jdump(p, o): | |
| p = pathlib.Path(p); p.parent.mkdir(parents=True, exist_ok=True); p.write_text(json.dumps(o, indent=2) + chr(10)) | |
| def host_gpus(): | |
| return [int(g) for g in (jload(HOSTS / (HOST + ".json"), {}) or {}).get("gpus", [])] | |
| def env_fp(): | |
| import platform | |
| fp = {"python": platform.python_version()} | |
| for m in ("numpy", "sympy"): | |
| try: fp[m] = __import__(m).__version__ | |
| except Exception: fp[m] = None | |
| glob = "/workspace/storage-shared/nlp/tungks" | |
| for p in sorted(pathlib.Path(glob).glob("simct-b200-portable*/scripts/evaluation/contract_eval.py")): | |
| fp["contract_sha"] = hashlib.sha256(p.read_bytes()).hexdigest()[:12]; break | |
| return fp | |
| def ckpt_sha(case, run, step): | |
| base = pathlib.Path(case) / "train" / run / "checkpoint" | |
| for cand in (base / ("step" + str(step)), base): | |
| if cand.is_dir(): | |
| files = sorted(f for f in cand.rglob("*") if f.is_file() and f.suffix in (".safetensors", ".json", ".bin")) | |
| if files: | |
| h = hashlib.sha256() | |
| for f in files[:64]: | |
| h.update(f.name.encode()); h.update(str(f.stat().st_size).encode()) | |
| return h.hexdigest()[:24] | |
| return "unknown:" + run + ":" + str(step) | |
| def bank_index(): | |
| f = BANK / "index.jsonl" | |
| if not f.is_file(): return [] | |
| out = [] | |
| for line in f.read_text().splitlines(): | |
| try: out.append(json.loads(line)) | |
| except Exception: pass | |
| return out | |
| def bank_lookup(ck, bench, seed): | |
| for r in bank_index(): | |
| if (r.get("ckpt_sha"), r.get("benchmark"), r.get("seed")) == (ck, bench, seed): return r | |
| return None | |
| def cell_paths(case, run, step, bench, seed): | |
| d = pathlib.Path(case) / "eval" / ("%s-step%d" % (run, step)) / "cells" | |
| return sorted(str(p) for p in d.glob("*/%s/%d" % (bench, seed))) | |
| def parse_steps(s): | |
| out = [] | |
| for part in s.split(","): | |
| if ".." in part: | |
| a, b = part.split("..") | |
| out += list(range(int(a), int(b) + 1, 40)) if int(b) - int(a) > 39 else list(range(int(a), int(b) + 1)) | |
| else: out.append(int(part)) | |
| return sorted(set(out)) | |
| def cmd_init(a): | |
| for sub in (Q/"pending", Q/"claimed", Q/"done", Q/"failed", BANK/"entries", LOCKS, RUN, HOSTS, BEAT): | |
| sub.mkdir(parents=True, exist_ok=True) | |
| (BANK/"index.jsonl").touch() | |
| print("EC_ROOT:", ROOT) | |
| print("host:", HOST, "| gpus:", host_gpus() or "(chua khai bao: ec host --gpus 0,1,3)") | |
| def cmd_run(a): | |
| """Khoi dong pool cho MAY NAY. --for/--until gioi han thoi gian nhan job (drain sau cell dang lam).""" | |
| runner = pathlib.Path(__file__).with_name("runners") / ("gen.sh" if a.type == "gen" else "score-cpu.sh") | |
| env = dict(os.environ) | |
| if a.__dict__.get("for_") or a.__dict__.get("until"): | |
| env["EC_UNTIL"] = str(a.__dict__.get("until") or "") | |
| env["EC_FOR"] = str(a.__dict__.get("for_") or "") | |
| env["EC_ROOT"] = str(ROOT); env["EC_HOST"] = HOST | |
| cmd = ["bash", str(runner)] + ([str(g).strip() for g in a.gpu.split(",")] if a.gpu else []) | |
| print("RUN", runner.name, "host", HOST, "gpu", a.gpu, "for", env.get("EC_FOR") or "-", "until", env.get("EC_UNTIL") or "-") | |
| if a.type == "gen": | |
| for g in [x.strip() for x in (a.gpu or "").split(",") if x.strip()]: | |
| os.spawnvp(os.P_NOWAIT, "bash", ["bash", str(runner), g, a.clone or ""]) | |
| print("da bat", len([x for x in (a.gpu or "").split(",") if x.strip()]), "runner gen (nen chay qua nohup/setsid)") | |
| else: | |
| os.spawnvp(os.P_NOWAIT, "bash", ["bash", str(runner), a.clone or ""]) | |
| print("da bat runner score-cpu") | |
| def cmd_stop(a): | |
| """Bao 1 (may, gpu) ngung nhan job: runner thay file nay se thoat sau cell dang lam.""" | |
| STOP.mkdir(parents=True, exist_ok=True) | |
| (STOP / ("%s-gpu-%s" % (a.host, a.gpu))).touch() | |
| print("STOP ->", a.host, "gpu", a.gpu, "| runner se thoat sau cell dang lam") | |
| def cmd_host(a): | |
| jdump(HOSTS / (HOST + ".json"), {"host": HOST, "gpus": [int(x) for x in a.gpus.split(",")], "declared": now()}) | |
| print("host", HOST, "gpus", a.gpus) | |
| def cmd_add(a): | |
| ckpt_dirs = list(getattr(a, "ckpt", []) or []) | |
| if getattr(a, "from_", None): | |
| for line in pathlib.Path(a.from_).read_text().splitlines(): | |
| line = line.strip() | |
| if line and not line.startswith("#"): ckpt_dirs.append(line) | |
| if ckpt_dirs: | |
| paths = [pathlib.Path(c) for c in ckpt_dirs] | |
| steps = [] | |
| for q in paths: | |
| name = q.name if q.name.startswith("step") else q.parent.name | |
| if name.startswith("step"): steps.append(int(name[4:])) | |
| q0 = paths[0] | |
| if q0.name.startswith("step"): # .../train/<run>/checkpoint/stepN | |
| run_dir, case_dir = q0.parents[1], q0.parents[3] | |
| else: # .../train/<run>/checkpoint | |
| run_dir, case_dir = q0.parents[0], q0.parents[2] | |
| a.case = str(case_dir) | |
| a.run = run_dir.name | |
| a.steps = ",".join(str(x) for x in sorted(set(steps))) | |
| print("tu --ckpt:", a.run, "steps", a.steps, "case", a.case) | |
| steps = parse_steps(a.steps) if a.steps else [] | |
| benches = BENCHES if a.bench == "all" else a.bench.split(",") | |
| seeds = [int(x) for x in a.seeds.split(",")] if a.seeds else [42, 43, 44] | |
| made = skip_bank = skip_cell = 0 | |
| print("host", HOST, "gpus", host_gpus()) | |
| for st in steps: | |
| ck = ckpt_sha(a.case, a.run, st) | |
| for b in benches: | |
| for sd in seeds: | |
| jid = "gen__%s__step%d__%s__seed%d" % (a.run, st, b, sd) | |
| if (Q / "pending" / (jid + ".json")).is_file() or (Q / "claimed" / (jid + ".json")).is_file(): continue | |
| if bank_lookup(ck, b, sd) and not a.force: skip_bank += 1; continue | |
| cells = cell_paths(a.case, a.run, st, b, sd) | |
| if cells and any((pathlib.Path(c) / "generation-complete.json").is_file() for c in cells) and not a.force: | |
| skip_cell += 1; continue | |
| jdump(Q / "pending" / (jid + ".json"), {"id": jid, "type": "gen", "case": str(a.case), | |
| "run": a.run, "step": st, "ckpt_sha": ck, "benchmark": b, "seed": sd, "gen_seed": 0, | |
| "cells": cells, "hosts_ok": host_gpus(), "needs": {"gpu": 1, "vram_gb": 140}, | |
| "engine": {"plan": str(pathlib.Path(a.case) / "eval" / ("%s-step%d" % (a.run, st)) / "plan.json")}, | |
| "bank_key": "pending:" + jid, "prio": 50, "created": now(), "by": "ec add@" + HOST, | |
| "claimed_by": None, "claimed_at": None}) | |
| made += 1 | |
| print("PENDING += %d | skip bank %d | skip cell da sinh %d" % (made, skip_bank, skip_cell)) | |
| def cmd_status(a): | |
| rows = {} | |
| for state in ("pending", "claimed", "done", "failed"): | |
| for f in (Q / state).glob("*.json"): | |
| j = jload(f, {}) or {} | |
| k = (j.get("run"), j.get("step")) | |
| rows.setdefault(k, {})[state] = rows.setdefault(k, {}).get(state, 0) + 1 | |
| print("%-18s %5s %8s %8s %6s %7s" % ("run", "step", "pending", "claimed", "done", "failed")) | |
| for k in sorted(rows, key=lambda x: (str(x[0]), x[1] or 0)): | |
| r = rows[k] | |
| print("%-18s %5s %8d %8d %6d %7d" % (k[0], k[1], r.get("pending", 0), r.get("claimed", 0), r.get("done", 0), r.get("failed", 0))) | |
| print("TONG:", {s: len(list((Q / s).glob("*.json"))) for s in ("pending", "claimed", "done", "failed")}) | |
| mine = host_gpus() | |
| print("MAY NAY:", HOST, "gpus", mine if mine else "(chua khai bao: ec host --gpus ...)") | |
| by_host = {} | |
| for f in (Q / "claimed").glob("*.json"): | |
| h = ((jload(f, {}) or {}).get("claimed_by") or "?").split(":")[0] | |
| by_host[h] = by_host.get(h, 0) + 1 | |
| if by_host: print("dang giu job theo may:", by_host) | |
| def cmd_why(a): | |
| for state in ("pending", "claimed", "done", "failed"): | |
| p = Q / state / (a.job + ".json") | |
| if not p.is_file(): continue | |
| j = jload(p, {}) or {} | |
| print("job:", a.job, "| state:", state, "| host", (j.get("claimed_by") or "-")) | |
| plan = pathlib.Path((j.get("engine") or {}).get("plan", "")) | |
| if not plan.is_file(): print(" LY DO: thieu plan ->", plan); return | |
| st = jload(plan.with_name("state.json")) | |
| if not st: print(" LY DO: plan chua chay lan nao"); return | |
| cur, qual = env_fp(), st.get("qualification") | |
| drift = [k for k in ("numpy", "sympy", "python") | |
| if qual and any(isinstance(q, dict) and q.get(k) not in (None, cur.get(k)) for q in qual)] | |
| print(" LY DO:", ("DRIFT moi truong: " + ",".join(drift)) if drift else "khong thay van de; xem run/" + a.job) | |
| return | |
| print("khong thay job", a.job) | |
| def cmd_beat(a): | |
| BEAT.mkdir(parents=True, exist_ok=True); (BEAT / a.job).touch(); print("beat", a.job, now()) | |
| def cmd_gc(a): | |
| unit = 1.0 if a.older_than.endswith("h") else 1.0/60 | |
| limit = time.time() - float(a.older_than[:-1]) * unit * 3600 | |
| moved = 0 | |
| for f in (Q / "claimed").glob("*.json"): | |
| beat = BEAT / f.stem | |
| if (beat.stat().st_mtime if beat.is_file() else f.stat().st_mtime) < limit: | |
| j = jload(f, {}) or {}; j["claimed_by"] = None; j["claimed_at"] = None | |
| jdump(Q / "pending" / f.name, j); f.unlink(); moved += 1 | |
| print("gc: tra ve pending", moved, "job") | |
| def cmd_bank(a): | |
| if a.action == "ls": | |
| idx = bank_index(); print("bank entries:", len(idx)) | |
| for r in idx[:6]: print(" ", r.get("ckpt_sha"), r.get("benchmark"), r.get("seed"), r.get("score"), "env", r.get("env_fp")) | |
| elif a.action == "import": | |
| n = 0 | |
| for m in sorted(pathlib.Path(a.target).glob("**/metrics.json")): | |
| d = jload(m, {}) or {} | |
| if "score" not in d: continue | |
| parts = m.parts | |
| run = next((x for x in parts if x.startswith("FIX-") or x.startswith("ALT-")), "?") | |
| step = next((int(x.split("step")[-1]) for x in parts if "-step" in x), None) | |
| try: | |
| i = parts.index("cells"); bench = parts[i + 2]; seed = int(parts[i + 3]) | |
| except Exception: bench, seed = "?", None | |
| ck = ckpt_sha(m.parents[5] if len(m.parents) > 5 else ".", run, step) if step is not None else "?" | |
| owner = next((p for p in m.parents if (p / "state.json").is_file()), None) | |
| envfp = "unknown:imported" | |
| if owner: | |
| q = (jload(owner / "state.json", {}) or {}).get("qualification") | |
| if q: envfp = "unknown:" + hashlib.sha256(json.dumps(q, sort_keys=True).encode()).hexdigest()[:12] | |
| # Khoa bank KHONG gom env_fp (quyet dinh 2026-10-02): doi moi truong cham khong tao ket qua moi, | |
| # nhung env_fp VAN duoc ghi vao entry de khong mat dau vet ai da cham. | |
| key = hashlib.sha256("|".join([ck, str(bench), str(seed)]).encode()).hexdigest()[:32] | |
| ent = {"key": key, "kind": "score", "ckpt_sha": ck, "run": run, "step": step, "benchmark": bench, | |
| "seed": seed, "score": d.get("score"), "count": d.get("count"), | |
| "truncated_fraction": d.get("truncated_fraction"), "contract_sha": str(d.get("contract"))[:12], | |
| "env_fp": envfp, "source": str(m), "created": now(), "host": HOST} | |
| jdump(BANK / "entries" / (key + ".json"), ent) | |
| with (BANK / "index.jsonl").open("a") as fh: fh.write(json.dumps(ent) + chr(10)) | |
| n += 1 | |
| print("bank import: +", n, "entry") | |
| else: | |
| lines = (BANK / "index.jsonl").read_text().splitlines() if (BANK / "index.jsonl").is_file() else [] | |
| bad = [1 for l in lines if "ckpt_sha" not in l or "env_fp" not in l] | |
| print("verify:", "OK" if not bad else "%d entry thieu ckpt_sha/env_fp" % len(bad)) | |
| def main(): | |
| ap = argparse.ArgumentParser(prog="ec"); sub = ap.add_subparsers(dest="cmd", required=True) | |
| p = sub.add_parser("init"); p.set_defaults(func=cmd_init) | |
| p = sub.add_parser("run"); p.add_argument("--type", default="gen", choices=["gen", "score-cpu"]) | |
| p.add_argument("--gpu", default=""); p.add_argument("--clone", default="") | |
| p.add_argument("--for", dest="for_", default=""); p.add_argument("--until", default="") | |
| p.set_defaults(func=cmd_run) | |
| p = sub.add_parser("stop"); p.add_argument("--host", required=True); p.add_argument("--gpu", required=True) | |
| p.set_defaults(func=cmd_stop) | |
| p = sub.add_parser("host"); p.add_argument("--gpus", required=True); p.set_defaults(func=cmd_host) | |
| p = sub.add_parser("add"); p.add_argument("--case", required=True); p.add_argument("--run", required=True) | |
| p.add_argument("--steps"); p.add_argument("--bench", default="all"); p.add_argument("--seeds") | |
| p.add_argument("--ckpt", action="append", default=[]) | |
| p.add_argument("--from", dest="from_", default=None) | |
| p.add_argument("--force", action="store_true"); p.set_defaults(func=cmd_add) | |
| p = sub.add_parser("status"); p.set_defaults(func=cmd_status) | |
| p = sub.add_parser("why"); p.add_argument("job"); p.set_defaults(func=cmd_why) | |
| p = sub.add_parser("beat"); p.add_argument("job"); p.set_defaults(func=cmd_beat) | |
| p = sub.add_parser("gc"); p.add_argument("--older-than", default="20m"); p.set_defaults(func=cmd_gc) | |
| p = sub.add_parser("bank"); p.add_argument("action", choices=["ls", "import", "verify"]) | |
| p.add_argument("target", nargs="?"); p.set_defaults(func=cmd_bank) | |
| a = ap.parse_args(); a.func(a) | |
| if __name__ == "__main__": main() | |