#!/usr/bin/env python3
"""eval-core CLI (v0.2). File-based, multi-machine, per-checkpoint.
ec host --gpus 0,1,3
ec add --case
--run R --steps 40..312 --bench all --seeds 42,43,44
ec status | ec why | ec beat | ec gc --older-than 20m
ec bank ls|import /eval|verify
"""
import argparse, datetime, hashlib, json, os, pathlib, socket, time
ROOT = pathlib.Path(os.environ.get("EC_ROOT", "/workspace/storage-shared/nlp/tungks/eval-core"))
HOST = os.environ.get("EC_HOST") or socket.gethostname().split(".")[0]
Q, BANK, LOCKS, RUN = ROOT/"queue", ROOT/"bank", ROOT/"locks", ROOT/"run"
HOSTS, BEAT = ROOT/"hosts", ROOT/"heartbeats"
STOP = ROOT/"stops"
BENCHES = ["gsm8k", "math500", "mbpp", "live-code-bench-v6"]
def now(): return datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds")
def jload(p, d=None):
try: return json.loads(pathlib.Path(p).read_text())
except Exception: return d
def jdump(p, o):
p = pathlib.Path(p); p.parent.mkdir(parents=True, exist_ok=True); p.write_text(json.dumps(o, indent=2) + chr(10))
def host_gpus():
return [int(g) for g in (jload(HOSTS / (HOST + ".json"), {}) or {}).get("gpus", [])]
def env_fp():
import platform
fp = {"python": platform.python_version()}
for m in ("numpy", "sympy"):
try: fp[m] = __import__(m).__version__
except Exception: fp[m] = None
glob = "/workspace/storage-shared/nlp/tungks"
for p in sorted(pathlib.Path(glob).glob("simct-b200-portable*/scripts/evaluation/contract_eval.py")):
fp["contract_sha"] = hashlib.sha256(p.read_bytes()).hexdigest()[:12]; break
return fp
def ckpt_sha(case, run, step):
base = pathlib.Path(case) / "train" / run / "checkpoint"
for cand in (base / ("step" + str(step)), base):
if cand.is_dir():
files = sorted(f for f in cand.rglob("*") if f.is_file() and f.suffix in (".safetensors", ".json", ".bin"))
if files:
h = hashlib.sha256()
for f in files[:64]:
h.update(f.name.encode()); h.update(str(f.stat().st_size).encode())
return h.hexdigest()[:24]
return "unknown:" + run + ":" + str(step)
def bank_index():
f = BANK / "index.jsonl"
if not f.is_file(): return []
out = []
for line in f.read_text().splitlines():
try: out.append(json.loads(line))
except Exception: pass
return out
def bank_lookup(ck, bench, seed):
for r in bank_index():
if (r.get("ckpt_sha"), r.get("benchmark"), r.get("seed")) == (ck, bench, seed): return r
return None
def cell_paths(case, run, step, bench, seed):
d = pathlib.Path(case) / "eval" / ("%s-step%d" % (run, step)) / "cells"
return sorted(str(p) for p in d.glob("*/%s/%d" % (bench, seed)))
def parse_steps(s):
out = []
for part in s.split(","):
if ".." in part:
a, b = part.split("..")
out += list(range(int(a), int(b) + 1, 40)) if int(b) - int(a) > 39 else list(range(int(a), int(b) + 1))
else: out.append(int(part))
return sorted(set(out))
def cmd_init(a):
for sub in (Q/"pending", Q/"claimed", Q/"done", Q/"failed", BANK/"entries", LOCKS, RUN, HOSTS, BEAT):
sub.mkdir(parents=True, exist_ok=True)
(BANK/"index.jsonl").touch()
print("EC_ROOT:", ROOT)
print("host:", HOST, "| gpus:", host_gpus() or "(chua khai bao: ec host --gpus 0,1,3)")
def cmd_run(a):
"""Khoi dong pool cho MAY NAY. --for/--until gioi han thoi gian nhan job (drain sau cell dang lam)."""
runner = pathlib.Path(__file__).with_name("runners") / ("gen.sh" if a.type == "gen" else "score-cpu.sh")
env = dict(os.environ)
if a.__dict__.get("for_") or a.__dict__.get("until"):
env["EC_UNTIL"] = str(a.__dict__.get("until") or "")
env["EC_FOR"] = str(a.__dict__.get("for_") or "")
env["EC_ROOT"] = str(ROOT); env["EC_HOST"] = HOST
cmd = ["bash", str(runner)] + ([str(g).strip() for g in a.gpu.split(",")] if a.gpu else [])
print("RUN", runner.name, "host", HOST, "gpu", a.gpu, "for", env.get("EC_FOR") or "-", "until", env.get("EC_UNTIL") or "-")
if a.type == "gen":
for g in [x.strip() for x in (a.gpu or "").split(",") if x.strip()]:
os.spawnvp(os.P_NOWAIT, "bash", ["bash", str(runner), g, a.clone or ""])
print("da bat", len([x for x in (a.gpu or "").split(",") if x.strip()]), "runner gen (nen chay qua nohup/setsid)")
else:
os.spawnvp(os.P_NOWAIT, "bash", ["bash", str(runner), a.clone or ""])
print("da bat runner score-cpu")
def cmd_stop(a):
"""Bao 1 (may, gpu) ngung nhan job: runner thay file nay se thoat sau cell dang lam."""
STOP.mkdir(parents=True, exist_ok=True)
(STOP / ("%s-gpu-%s" % (a.host, a.gpu))).touch()
print("STOP ->", a.host, "gpu", a.gpu, "| runner se thoat sau cell dang lam")
def cmd_host(a):
jdump(HOSTS / (HOST + ".json"), {"host": HOST, "gpus": [int(x) for x in a.gpus.split(",")], "declared": now()})
print("host", HOST, "gpus", a.gpus)
def cmd_add(a):
ckpt_dirs = list(getattr(a, "ckpt", []) or [])
if getattr(a, "from_", None):
for line in pathlib.Path(a.from_).read_text().splitlines():
line = line.strip()
if line and not line.startswith("#"): ckpt_dirs.append(line)
if ckpt_dirs:
paths = [pathlib.Path(c) for c in ckpt_dirs]
steps = []
for q in paths:
name = q.name if q.name.startswith("step") else q.parent.name
if name.startswith("step"): steps.append(int(name[4:]))
q0 = paths[0]
if q0.name.startswith("step"): # .../train//checkpoint/stepN
run_dir, case_dir = q0.parents[1], q0.parents[3]
else: # .../train//checkpoint
run_dir, case_dir = q0.parents[0], q0.parents[2]
a.case = str(case_dir)
a.run = run_dir.name
a.steps = ",".join(str(x) for x in sorted(set(steps)))
print("tu --ckpt:", a.run, "steps", a.steps, "case", a.case)
steps = parse_steps(a.steps) if a.steps else []
benches = BENCHES if a.bench == "all" else a.bench.split(",")
seeds = [int(x) for x in a.seeds.split(",")] if a.seeds else [42, 43, 44]
made = skip_bank = skip_cell = 0
print("host", HOST, "gpus", host_gpus())
for st in steps:
ck = ckpt_sha(a.case, a.run, st)
for b in benches:
for sd in seeds:
jid = "gen__%s__step%d__%s__seed%d" % (a.run, st, b, sd)
if (Q / "pending" / (jid + ".json")).is_file() or (Q / "claimed" / (jid + ".json")).is_file(): continue
if bank_lookup(ck, b, sd) and not a.force: skip_bank += 1; continue
cells = cell_paths(a.case, a.run, st, b, sd)
if cells and any((pathlib.Path(c) / "generation-complete.json").is_file() for c in cells) and not a.force:
skip_cell += 1; continue
jdump(Q / "pending" / (jid + ".json"), {"id": jid, "type": "gen", "case": str(a.case),
"run": a.run, "step": st, "ckpt_sha": ck, "benchmark": b, "seed": sd, "gen_seed": 0,
"cells": cells, "hosts_ok": host_gpus(), "needs": {"gpu": 1, "vram_gb": 140},
"engine": {"plan": str(pathlib.Path(a.case) / "eval" / ("%s-step%d" % (a.run, st)) / "plan.json")},
"bank_key": "pending:" + jid, "prio": 50, "created": now(), "by": "ec add@" + HOST,
"claimed_by": None, "claimed_at": None})
made += 1
print("PENDING += %d | skip bank %d | skip cell da sinh %d" % (made, skip_bank, skip_cell))
def cmd_status(a):
rows = {}
for state in ("pending", "claimed", "done", "failed"):
for f in (Q / state).glob("*.json"):
j = jload(f, {}) or {}
k = (j.get("run"), j.get("step"))
rows.setdefault(k, {})[state] = rows.setdefault(k, {}).get(state, 0) + 1
print("%-18s %5s %8s %8s %6s %7s" % ("run", "step", "pending", "claimed", "done", "failed"))
for k in sorted(rows, key=lambda x: (str(x[0]), x[1] or 0)):
r = rows[k]
print("%-18s %5s %8d %8d %6d %7d" % (k[0], k[1], r.get("pending", 0), r.get("claimed", 0), r.get("done", 0), r.get("failed", 0)))
print("TONG:", {s: len(list((Q / s).glob("*.json"))) for s in ("pending", "claimed", "done", "failed")})
mine = host_gpus()
print("MAY NAY:", HOST, "gpus", mine if mine else "(chua khai bao: ec host --gpus ...)")
by_host = {}
for f in (Q / "claimed").glob("*.json"):
h = ((jload(f, {}) or {}).get("claimed_by") or "?").split(":")[0]
by_host[h] = by_host.get(h, 0) + 1
if by_host: print("dang giu job theo may:", by_host)
def cmd_why(a):
for state in ("pending", "claimed", "done", "failed"):
p = Q / state / (a.job + ".json")
if not p.is_file(): continue
j = jload(p, {}) or {}
print("job:", a.job, "| state:", state, "| host", (j.get("claimed_by") or "-"))
plan = pathlib.Path((j.get("engine") or {}).get("plan", ""))
if not plan.is_file(): print(" LY DO: thieu plan ->", plan); return
st = jload(plan.with_name("state.json"))
if not st: print(" LY DO: plan chua chay lan nao"); return
cur, qual = env_fp(), st.get("qualification")
drift = [k for k in ("numpy", "sympy", "python")
if qual and any(isinstance(q, dict) and q.get(k) not in (None, cur.get(k)) for q in qual)]
print(" LY DO:", ("DRIFT moi truong: " + ",".join(drift)) if drift else "khong thay van de; xem run/" + a.job)
return
print("khong thay job", a.job)
def cmd_beat(a):
BEAT.mkdir(parents=True, exist_ok=True); (BEAT / a.job).touch(); print("beat", a.job, now())
def cmd_gc(a):
unit = 1.0 if a.older_than.endswith("h") else 1.0/60
limit = time.time() - float(a.older_than[:-1]) * unit * 3600
moved = 0
for f in (Q / "claimed").glob("*.json"):
beat = BEAT / f.stem
if (beat.stat().st_mtime if beat.is_file() else f.stat().st_mtime) < limit:
j = jload(f, {}) or {}; j["claimed_by"] = None; j["claimed_at"] = None
jdump(Q / "pending" / f.name, j); f.unlink(); moved += 1
print("gc: tra ve pending", moved, "job")
def cmd_bank(a):
if a.action == "ls":
idx = bank_index(); print("bank entries:", len(idx))
for r in idx[:6]: print(" ", r.get("ckpt_sha"), r.get("benchmark"), r.get("seed"), r.get("score"), "env", r.get("env_fp"))
elif a.action == "import":
n = 0
for m in sorted(pathlib.Path(a.target).glob("**/metrics.json")):
d = jload(m, {}) or {}
if "score" not in d: continue
parts = m.parts
run = next((x for x in parts if x.startswith("FIX-") or x.startswith("ALT-")), "?")
step = next((int(x.split("step")[-1]) for x in parts if "-step" in x), None)
try:
i = parts.index("cells"); bench = parts[i + 2]; seed = int(parts[i + 3])
except Exception: bench, seed = "?", None
ck = ckpt_sha(m.parents[5] if len(m.parents) > 5 else ".", run, step) if step is not None else "?"
owner = next((p for p in m.parents if (p / "state.json").is_file()), None)
envfp = "unknown:imported"
if owner:
q = (jload(owner / "state.json", {}) or {}).get("qualification")
if q: envfp = "unknown:" + hashlib.sha256(json.dumps(q, sort_keys=True).encode()).hexdigest()[:12]
# Khoa bank KHONG gom env_fp (quyet dinh 2026-10-02): doi moi truong cham khong tao ket qua moi,
# nhung env_fp VAN duoc ghi vao entry de khong mat dau vet ai da cham.
key = hashlib.sha256("|".join([ck, str(bench), str(seed)]).encode()).hexdigest()[:32]
ent = {"key": key, "kind": "score", "ckpt_sha": ck, "run": run, "step": step, "benchmark": bench,
"seed": seed, "score": d.get("score"), "count": d.get("count"),
"truncated_fraction": d.get("truncated_fraction"), "contract_sha": str(d.get("contract"))[:12],
"env_fp": envfp, "source": str(m), "created": now(), "host": HOST}
jdump(BANK / "entries" / (key + ".json"), ent)
with (BANK / "index.jsonl").open("a") as fh: fh.write(json.dumps(ent) + chr(10))
n += 1
print("bank import: +", n, "entry")
else:
lines = (BANK / "index.jsonl").read_text().splitlines() if (BANK / "index.jsonl").is_file() else []
bad = [1 for l in lines if "ckpt_sha" not in l or "env_fp" not in l]
print("verify:", "OK" if not bad else "%d entry thieu ckpt_sha/env_fp" % len(bad))
def main():
ap = argparse.ArgumentParser(prog="ec"); sub = ap.add_subparsers(dest="cmd", required=True)
p = sub.add_parser("init"); p.set_defaults(func=cmd_init)
p = sub.add_parser("run"); p.add_argument("--type", default="gen", choices=["gen", "score-cpu"])
p.add_argument("--gpu", default=""); p.add_argument("--clone", default="")
p.add_argument("--for", dest="for_", default=""); p.add_argument("--until", default="")
p.set_defaults(func=cmd_run)
p = sub.add_parser("stop"); p.add_argument("--host", required=True); p.add_argument("--gpu", required=True)
p.set_defaults(func=cmd_stop)
p = sub.add_parser("host"); p.add_argument("--gpus", required=True); p.set_defaults(func=cmd_host)
p = sub.add_parser("add"); p.add_argument("--case", required=True); p.add_argument("--run", required=True)
p.add_argument("--steps"); p.add_argument("--bench", default="all"); p.add_argument("--seeds")
p.add_argument("--ckpt", action="append", default=[])
p.add_argument("--from", dest="from_", default=None)
p.add_argument("--force", action="store_true"); p.set_defaults(func=cmd_add)
p = sub.add_parser("status"); p.set_defaults(func=cmd_status)
p = sub.add_parser("why"); p.add_argument("job"); p.set_defaults(func=cmd_why)
p = sub.add_parser("beat"); p.add_argument("job"); p.set_defaults(func=cmd_beat)
p = sub.add_parser("gc"); p.add_argument("--older-than", default="20m"); p.set_defaults(func=cmd_gc)
p = sub.add_parser("bank"); p.add_argument("action", choices=["ls", "import", "verify"])
p.add_argument("target", nargs="?"); p.set_defaults(func=cmd_bank)
a = ap.parse_args(); a.func(a)
if __name__ == "__main__": main()