#!/usr/bin/env python3 """eval-core CLI (v0.2). File-based, multi-machine, per-checkpoint. ec host --gpus 0,1,3 ec add --case --run R --steps 40..312 --bench all --seeds 42,43,44 ec status | ec why | ec beat | ec gc --older-than 20m ec bank ls|import /eval|verify """ import argparse, datetime, hashlib, json, os, pathlib, socket, time ROOT = pathlib.Path(os.environ.get("EC_ROOT", "/workspace/storage-shared/nlp/tungks/eval-core")) HOST = os.environ.get("EC_HOST") or socket.gethostname().split(".")[0] Q, BANK, LOCKS, RUN = ROOT/"queue", ROOT/"bank", ROOT/"locks", ROOT/"run" HOSTS, BEAT = ROOT/"hosts", ROOT/"heartbeats" STOP = ROOT/"stops" BENCHES = ["gsm8k", "math500", "mbpp", "live-code-bench-v6"] def now(): return datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds") def jload(p, d=None): try: return json.loads(pathlib.Path(p).read_text()) except Exception: return d def jdump(p, o): p = pathlib.Path(p); p.parent.mkdir(parents=True, exist_ok=True); p.write_text(json.dumps(o, indent=2) + chr(10)) def host_gpus(): return [int(g) for g in (jload(HOSTS / (HOST + ".json"), {}) or {}).get("gpus", [])] def env_fp(): import platform fp = {"python": platform.python_version()} for m in ("numpy", "sympy"): try: fp[m] = __import__(m).__version__ except Exception: fp[m] = None glob = "/workspace/storage-shared/nlp/tungks" for p in sorted(pathlib.Path(glob).glob("simct-b200-portable*/scripts/evaluation/contract_eval.py")): fp["contract_sha"] = hashlib.sha256(p.read_bytes()).hexdigest()[:12]; break return fp def ckpt_sha(case, run, step): base = pathlib.Path(case) / "train" / run / "checkpoint" for cand in (base / ("step" + str(step)), base): if cand.is_dir(): files = sorted(f for f in cand.rglob("*") if f.is_file() and f.suffix in (".safetensors", ".json", ".bin")) if files: h = hashlib.sha256() for f in files[:64]: h.update(f.name.encode()); h.update(str(f.stat().st_size).encode()) return h.hexdigest()[:24] return "unknown:" + run + ":" + str(step) def bank_index(): f = BANK / "index.jsonl" if not f.is_file(): return [] out = [] for line in f.read_text().splitlines(): try: out.append(json.loads(line)) except Exception: pass return out def bank_lookup(ck, bench, seed): for r in bank_index(): if (r.get("ckpt_sha"), r.get("benchmark"), r.get("seed")) == (ck, bench, seed): return r return None def cell_paths(case, run, step, bench, seed): d = pathlib.Path(case) / "eval" / ("%s-step%d" % (run, step)) / "cells" return sorted(str(p) for p in d.glob("*/%s/%d" % (bench, seed))) def parse_steps(s): out = [] for part in s.split(","): if ".." in part: a, b = part.split("..") out += list(range(int(a), int(b) + 1, 40)) if int(b) - int(a) > 39 else list(range(int(a), int(b) + 1)) else: out.append(int(part)) return sorted(set(out)) def cmd_init(a): for sub in (Q/"pending", Q/"claimed", Q/"done", Q/"failed", BANK/"entries", LOCKS, RUN, HOSTS, BEAT): sub.mkdir(parents=True, exist_ok=True) (BANK/"index.jsonl").touch() print("EC_ROOT:", ROOT) print("host:", HOST, "| gpus:", host_gpus() or "(chua khai bao: ec host --gpus 0,1,3)") def cmd_run(a): """Khoi dong pool cho MAY NAY. --for/--until gioi han thoi gian nhan job (drain sau cell dang lam).""" runner = pathlib.Path(__file__).with_name("runners") / ("gen.sh" if a.type == "gen" else "score-cpu.sh") env = dict(os.environ) if a.__dict__.get("for_") or a.__dict__.get("until"): env["EC_UNTIL"] = str(a.__dict__.get("until") or "") env["EC_FOR"] = str(a.__dict__.get("for_") or "") env["EC_ROOT"] = str(ROOT); env["EC_HOST"] = HOST cmd = ["bash", str(runner)] + ([str(g).strip() for g in a.gpu.split(",")] if a.gpu else []) print("RUN", runner.name, "host", HOST, "gpu", a.gpu, "for", env.get("EC_FOR") or "-", "until", env.get("EC_UNTIL") or "-") if a.type == "gen": for g in [x.strip() for x in (a.gpu or "").split(",") if x.strip()]: os.spawnvp(os.P_NOWAIT, "bash", ["bash", str(runner), g, a.clone or ""]) print("da bat", len([x for x in (a.gpu or "").split(",") if x.strip()]), "runner gen (nen chay qua nohup/setsid)") else: os.spawnvp(os.P_NOWAIT, "bash", ["bash", str(runner), a.clone or ""]) print("da bat runner score-cpu") def cmd_stop(a): """Bao 1 (may, gpu) ngung nhan job: runner thay file nay se thoat sau cell dang lam.""" STOP.mkdir(parents=True, exist_ok=True) (STOP / ("%s-gpu-%s" % (a.host, a.gpu))).touch() print("STOP ->", a.host, "gpu", a.gpu, "| runner se thoat sau cell dang lam") def cmd_host(a): jdump(HOSTS / (HOST + ".json"), {"host": HOST, "gpus": [int(x) for x in a.gpus.split(",")], "declared": now()}) print("host", HOST, "gpus", a.gpus) def cmd_add(a): ckpt_dirs = list(getattr(a, "ckpt", []) or []) if getattr(a, "from_", None): for line in pathlib.Path(a.from_).read_text().splitlines(): line = line.strip() if line and not line.startswith("#"): ckpt_dirs.append(line) if ckpt_dirs: paths = [pathlib.Path(c) for c in ckpt_dirs] steps = [] for q in paths: name = q.name if q.name.startswith("step") else q.parent.name if name.startswith("step"): steps.append(int(name[4:])) q0 = paths[0] if q0.name.startswith("step"): # .../train//checkpoint/stepN run_dir, case_dir = q0.parents[1], q0.parents[3] else: # .../train//checkpoint run_dir, case_dir = q0.parents[0], q0.parents[2] a.case = str(case_dir) a.run = run_dir.name a.steps = ",".join(str(x) for x in sorted(set(steps))) print("tu --ckpt:", a.run, "steps", a.steps, "case", a.case) steps = parse_steps(a.steps) if a.steps else [] benches = BENCHES if a.bench == "all" else a.bench.split(",") seeds = [int(x) for x in a.seeds.split(",")] if a.seeds else [42, 43, 44] made = skip_bank = skip_cell = 0 print("host", HOST, "gpus", host_gpus()) for st in steps: ck = ckpt_sha(a.case, a.run, st) for b in benches: for sd in seeds: jid = "gen__%s__step%d__%s__seed%d" % (a.run, st, b, sd) if (Q / "pending" / (jid + ".json")).is_file() or (Q / "claimed" / (jid + ".json")).is_file(): continue if bank_lookup(ck, b, sd) and not a.force: skip_bank += 1; continue cells = cell_paths(a.case, a.run, st, b, sd) if cells and any((pathlib.Path(c) / "generation-complete.json").is_file() for c in cells) and not a.force: skip_cell += 1; continue jdump(Q / "pending" / (jid + ".json"), {"id": jid, "type": "gen", "case": str(a.case), "run": a.run, "step": st, "ckpt_sha": ck, "benchmark": b, "seed": sd, "gen_seed": 0, "cells": cells, "hosts_ok": host_gpus(), "needs": {"gpu": 1, "vram_gb": 140}, "engine": {"plan": str(pathlib.Path(a.case) / "eval" / ("%s-step%d" % (a.run, st)) / "plan.json")}, "bank_key": "pending:" + jid, "prio": 50, "created": now(), "by": "ec add@" + HOST, "claimed_by": None, "claimed_at": None}) made += 1 print("PENDING += %d | skip bank %d | skip cell da sinh %d" % (made, skip_bank, skip_cell)) def cmd_status(a): rows = {} for state in ("pending", "claimed", "done", "failed"): for f in (Q / state).glob("*.json"): j = jload(f, {}) or {} k = (j.get("run"), j.get("step")) rows.setdefault(k, {})[state] = rows.setdefault(k, {}).get(state, 0) + 1 print("%-18s %5s %8s %8s %6s %7s" % ("run", "step", "pending", "claimed", "done", "failed")) for k in sorted(rows, key=lambda x: (str(x[0]), x[1] or 0)): r = rows[k] print("%-18s %5s %8d %8d %6d %7d" % (k[0], k[1], r.get("pending", 0), r.get("claimed", 0), r.get("done", 0), r.get("failed", 0))) print("TONG:", {s: len(list((Q / s).glob("*.json"))) for s in ("pending", "claimed", "done", "failed")}) mine = host_gpus() print("MAY NAY:", HOST, "gpus", mine if mine else "(chua khai bao: ec host --gpus ...)") by_host = {} for f in (Q / "claimed").glob("*.json"): h = ((jload(f, {}) or {}).get("claimed_by") or "?").split(":")[0] by_host[h] = by_host.get(h, 0) + 1 if by_host: print("dang giu job theo may:", by_host) def cmd_why(a): for state in ("pending", "claimed", "done", "failed"): p = Q / state / (a.job + ".json") if not p.is_file(): continue j = jload(p, {}) or {} print("job:", a.job, "| state:", state, "| host", (j.get("claimed_by") or "-")) plan = pathlib.Path((j.get("engine") or {}).get("plan", "")) if not plan.is_file(): print(" LY DO: thieu plan ->", plan); return st = jload(plan.with_name("state.json")) if not st: print(" LY DO: plan chua chay lan nao"); return cur, qual = env_fp(), st.get("qualification") drift = [k for k in ("numpy", "sympy", "python") if qual and any(isinstance(q, dict) and q.get(k) not in (None, cur.get(k)) for q in qual)] print(" LY DO:", ("DRIFT moi truong: " + ",".join(drift)) if drift else "khong thay van de; xem run/" + a.job) return print("khong thay job", a.job) def cmd_beat(a): BEAT.mkdir(parents=True, exist_ok=True); (BEAT / a.job).touch(); print("beat", a.job, now()) def cmd_gc(a): unit = 1.0 if a.older_than.endswith("h") else 1.0/60 limit = time.time() - float(a.older_than[:-1]) * unit * 3600 moved = 0 for f in (Q / "claimed").glob("*.json"): beat = BEAT / f.stem if (beat.stat().st_mtime if beat.is_file() else f.stat().st_mtime) < limit: j = jload(f, {}) or {}; j["claimed_by"] = None; j["claimed_at"] = None jdump(Q / "pending" / f.name, j); f.unlink(); moved += 1 print("gc: tra ve pending", moved, "job") def cmd_bank(a): if a.action == "ls": idx = bank_index(); print("bank entries:", len(idx)) for r in idx[:6]: print(" ", r.get("ckpt_sha"), r.get("benchmark"), r.get("seed"), r.get("score"), "env", r.get("env_fp")) elif a.action == "import": n = 0 for m in sorted(pathlib.Path(a.target).glob("**/metrics.json")): d = jload(m, {}) or {} if "score" not in d: continue parts = m.parts run = next((x for x in parts if x.startswith("FIX-") or x.startswith("ALT-")), "?") step = next((int(x.split("step")[-1]) for x in parts if "-step" in x), None) try: i = parts.index("cells"); bench = parts[i + 2]; seed = int(parts[i + 3]) except Exception: bench, seed = "?", None ck = ckpt_sha(m.parents[5] if len(m.parents) > 5 else ".", run, step) if step is not None else "?" owner = next((p for p in m.parents if (p / "state.json").is_file()), None) envfp = "unknown:imported" if owner: q = (jload(owner / "state.json", {}) or {}).get("qualification") if q: envfp = "unknown:" + hashlib.sha256(json.dumps(q, sort_keys=True).encode()).hexdigest()[:12] # Khoa bank KHONG gom env_fp (quyet dinh 2026-10-02): doi moi truong cham khong tao ket qua moi, # nhung env_fp VAN duoc ghi vao entry de khong mat dau vet ai da cham. key = hashlib.sha256("|".join([ck, str(bench), str(seed)]).encode()).hexdigest()[:32] ent = {"key": key, "kind": "score", "ckpt_sha": ck, "run": run, "step": step, "benchmark": bench, "seed": seed, "score": d.get("score"), "count": d.get("count"), "truncated_fraction": d.get("truncated_fraction"), "contract_sha": str(d.get("contract"))[:12], "env_fp": envfp, "source": str(m), "created": now(), "host": HOST} jdump(BANK / "entries" / (key + ".json"), ent) with (BANK / "index.jsonl").open("a") as fh: fh.write(json.dumps(ent) + chr(10)) n += 1 print("bank import: +", n, "entry") else: lines = (BANK / "index.jsonl").read_text().splitlines() if (BANK / "index.jsonl").is_file() else [] bad = [1 for l in lines if "ckpt_sha" not in l or "env_fp" not in l] print("verify:", "OK" if not bad else "%d entry thieu ckpt_sha/env_fp" % len(bad)) def main(): ap = argparse.ArgumentParser(prog="ec"); sub = ap.add_subparsers(dest="cmd", required=True) p = sub.add_parser("init"); p.set_defaults(func=cmd_init) p = sub.add_parser("run"); p.add_argument("--type", default="gen", choices=["gen", "score-cpu"]) p.add_argument("--gpu", default=""); p.add_argument("--clone", default="") p.add_argument("--for", dest="for_", default=""); p.add_argument("--until", default="") p.set_defaults(func=cmd_run) p = sub.add_parser("stop"); p.add_argument("--host", required=True); p.add_argument("--gpu", required=True) p.set_defaults(func=cmd_stop) p = sub.add_parser("host"); p.add_argument("--gpus", required=True); p.set_defaults(func=cmd_host) p = sub.add_parser("add"); p.add_argument("--case", required=True); p.add_argument("--run", required=True) p.add_argument("--steps"); p.add_argument("--bench", default="all"); p.add_argument("--seeds") p.add_argument("--ckpt", action="append", default=[]) p.add_argument("--from", dest="from_", default=None) p.add_argument("--force", action="store_true"); p.set_defaults(func=cmd_add) p = sub.add_parser("status"); p.set_defaults(func=cmd_status) p = sub.add_parser("why"); p.add_argument("job"); p.set_defaults(func=cmd_why) p = sub.add_parser("beat"); p.add_argument("job"); p.set_defaults(func=cmd_beat) p = sub.add_parser("gc"); p.add_argument("--older-than", default="20m"); p.set_defaults(func=cmd_gc) p = sub.add_parser("bank"); p.add_argument("action", choices=["ls", "import", "verify"]) p.add_argument("target", nargs="?"); p.set_defaults(func=cmd_bank) a = ap.parse_args(); a.func(a) if __name__ == "__main__": main()