progress / artifacts /scripts /hf_push.py
eyad-silx's picture
push artifacts/scripts/hf_push.py
95aa2b3 verified
Raw History Blame
6.68 kB
#!/usr/bin/env python3
"""Push Ingot (SN97) build-pod artifacts to HF repo eyad-silx/progress.
Modes: --small (docs/logs/scripts), --big (GGUFs + QAT states), default = both.
Only uploads new/changed files (size+mtime_ns fingerprint), records sha256 in
state for the MANIFEST. State: /root/.hf_push_state.json. Log: /root/hf_push.log.
"""
import argparse, glob, hashlib, json, os, sys
from datetime import datetime, timezone
from pathlib import Path
from huggingface_hub import HfApi
REPO = "eyad-silx/progress"
TOKEN = Path("/root/.hf_token").read_text().strip()
STATE_P = Path("/root/.hf_push_state.json")
LOG_P = Path("/root/hf_push.log")
README = """# Ingot (SN97) challenger build β€” progress proof
Automated backup of build/train/eval artifacts from the H100 pod (RunPod, 1x H100 NVL).
Pushed by `hf_push.py` β€” checks hourly and uploads any new/changed artifact.
- `king-ix/` β€” incumbent king GGUF (Ingot-IX-27B-Q2) + provenance files (baseline)
- `chal-identity/`, `chal-v1/`, `chal-n2/`, `chal-norms/` β€” challenger GGUF iterations
(scale-refit / QAT variants; same 27B Q2 architecture as the king)
- `states/` β€” QAT state checkpoints (`t0..tN.safetensors`)
- `artifacts/` β€” `HANDOFF.md`, build/train/eval scripts, run logs, sample draws,
measurement results (`measure*.jsonl`, `pick*.jsonl`, `drawn.jsonl`)
- `MANIFEST.md` β€” size + sha256 + upload time of every artifact (regenerated each push)
- `PROGRESS.md` β€” push history
The GGUFs are ~6.8 GB each (PQ2_0 quantized 27B linear-attention models).
"""
def now():
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def log(evt, **kw):
line = json.dumps({"t": now(), "event": evt, **kw})
with open(LOG_P, "a") as f:
f.write(line + "\n")
print(line, flush=True)
def sha256(p):
h = hashlib.sha256()
with open(p, "rb") as f:
for chunk in iter(lambda: f.read(1 << 24), b""):
h.update(chunk)
return h.hexdigest()
def load_state():
return json.loads(STATE_P.read_text()) if STATE_P.exists() else {}
def save_state(s):
STATE_P.write_text(json.dumps(s, indent=1, sort_keys=True))
def small_tracks():
t = [("/root/HANDOFF.md", "artifacts/HANDOFF.md")]
for pat, dest in [("/root/*.py", "artifacts/scripts/"),
("/root/*.sh", "artifacts/scripts/"),
("/root/*.jsonl", "artifacts/"),
("/root/*.json", "artifacts/"),
("/root/*.log", "artifacts/logs/")]:
for p in sorted(glob.glob(pat)):
name = os.path.basename(p)
if name in ("hf_push.log", "hf_push_nohup.log"):
continue
t.append((p, dest + name))
for extra in ("README.md", "LICENSE", "NOTICE.txt"):
p = f"/root/models/king-ix/{extra}"
if os.path.exists(p):
t.append((p, f"king-ix/{extra}"))
return t
def big_tracks():
# priority order: newest challenger first (pod lifetime is uncertain)
t = [("/root/models/chal-identity/Ingot-IX-27B-Q2.gguf", "chal-identity/Ingot-IX-27B-Q2.gguf"),
("/root/models/chal-v1/chal-v1.gguf", "chal-v1/chal-v1.gguf"),
("/root/models/chal-n2/chal-norms.gguf", "chal-n2/chal-norms.gguf"),
("/root/models/chal-norms/chal-norms.gguf", "chal-norms/chal-norms.gguf"),
("/root/models/king-ix/Ingot-IX-27B-Q2.gguf", "king-ix/Ingot-IX-27B-Q2.gguf")]
for p in sorted(glob.glob("/root/states/*.safetensors")):
t.append((p, "states/" + os.path.basename(p)))
return t
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--small", action="store_true")
ap.add_argument("--big", action="store_true")
args = ap.parse_args()
if args.small or args.big:
tracks = (small_tracks() if args.small else []) + (big_tracks() if args.big else [])
else:
tracks = small_tracks() + big_tracks()
state = load_state()
api = HfApi(token=TOKEN)
results = []
for local, repo in tracks:
if not os.path.exists(local):
continue
st = os.stat(local)
fp = f"{st.st_size}:{st.st_mtime_ns}"
prev = state.get(repo)
if prev and prev["fp"] == fp:
results.append((repo, "unchanged"))
continue
try:
d = sha256(local)
api.upload_file(path_or_fileobj=local, path_in_repo=repo, repo_id=REPO,
repo_type="model", commit_message=f"push {repo}")
state[repo] = {"fp": fp, "sha256": d, "size": st.st_size, "uploaded_at": now()}
save_state(state)
log("uploaded", local=local, repo=repo, sha256=d, size=st.st_size)
results.append((repo, "uploaded"))
except Exception as e:
log("error", local=local, repo=repo, err=repr(e)[:500])
results.append((repo, "FAILED"))
# regenerate docs (manifest / progress / readme) from state
try:
manifest = ["# Manifest β€” eyad-silx/progress", "",
"| repo path | size (bytes) | sha256 | uploaded (UTC) |",
"|---|---|---|---|"]
for repo in sorted(state):
v = state[repo]
manifest.append(f"| `{repo}` | {v['size']:,} | `{v['sha256']}` | {v['uploaded_at']} |")
Path("/tmp/MANIFEST.md").write_text("\n".join(manifest) + "\n")
api.upload_file(path_or_fileobj="/tmp/MANIFEST.md", path_in_repo="MANIFEST.md",
repo_id=REPO, repo_type="model", commit_message="update MANIFEST.md")
hist = [json.loads(l) for l in LOG_P.read_text().splitlines()[-80:] if l.strip()]
prog = ["# Progress β€” push history", "", f"Last run: {now()}", ""]
prog += [f"- `{h['t']}` **{h['event']}** {h.get('repo', h.get('local', ''))}" for h in hist]
Path("/tmp/PROGRESS.md").write_text("\n".join(prog) + "\n")
api.upload_file(path_or_fileobj="/tmp/PROGRESS.md", path_in_repo="PROGRESS.md",
repo_id=REPO, repo_type="model", commit_message="update PROGRESS.md")
Path("/tmp/README_REPO.md").write_text(README)
api.upload_file(path_or_fileobj="/tmp/README_REPO.md", path_in_repo="README.md",
repo_id=REPO, repo_type="model", commit_message="README")
except Exception as e:
log("error", step="docs", err=repr(e)[:500])
n_up = sum(1 for r in results if r[1] == "uploaded")
n_un = sum(1 for r in results if r[1] == "unchanged")
n_fail = sum(1 for r in results if r[1] == "FAILED")
log("run_done", uploaded=n_up, unchanged=n_un, failed=n_fail)
sys.exit(1 if n_fail else 0)
if __name__ == "__main__":
main()