File size: 1,803 Bytes
519e2ef | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 | #!/usr/bin/env python3
"""Checker — push HF sous v3_scale/ uniquement (thefinalboss/SPIRA). Jamais de delete, jamais d'écriture hors v3_scale/.
Usage : hf_push_new.py "message" local1=repo/path1 local2=repo/path2 ...
- refuse tout chemin repo hors v3_scale/ ; refuse d'écraser un checkpoint (.npz) déjà présent sur le repo.
- refuse les données d'éval (hits*.npz, samples/, prompts_v1.json, results_gen*, corpus_gutenberg).
- log : checker/scale/hf_push.log (heure Toronto, URL commit, fichiers+tailles)."""
import os, sys, datetime, zoneinfo
from huggingface_hub import HfApi, CommitOperationAdd
REPO = "thefinalboss/SPIRA"; LOG = "/workspace/spira_shared/checker/scale/hf_push.log"
BAN = ("hits", "/samples/", "prompts_v1.json", "results_gen", "corpus_gutenberg", ".env", "token")
def main():
msg, pairs = sys.argv[1], [a.split("=", 1) for a in sys.argv[2:]]
api = HfApi(); remote = set(api.list_repo_files(REPO)); ops, lst = [], []
for loc, rp in pairs:
assert rp.startswith("v3_scale/") and ".." not in rp, rp
assert not any(b in loc or b in rp for b in BAN), f"interdit: {loc}"
if rp.endswith(".npz") and rp in remote: print("déjà présent, ignoré:", rp); continue
ops.append(CommitOperationAdd(path_in_repo=rp, path_or_fileobj=loc)); lst.append((rp, os.path.getsize(loc)))
if not ops: print("rien à pousser"); return
ci = api.create_commit(REPO, operations=ops, commit_message=msg)
t = datetime.datetime.now(zoneinfo.ZoneInfo("America/Toronto")).strftime("%F %H:%M:%S %Z")
with open(LOG, "a") as f:
f.write(f"{t}\t{ci.oid}\t{ci.commit_url}\t{msg}\n")
for rp, s in lst: f.write(f" {rp}\t{s}\n")
print(t, ci.oid, ci.commit_url); [print(" ", rp, s) for rp, s in lst]
if __name__ == "__main__": main()
|