Cion-lab commited on
Commit
9a7dcc5
·
verified ·
1 Parent(s): 5f6c3e9

train: preflight -- download_checkpoint prefix/landing-dir fix (found by P1 v2)

Browse files
Files changed (1) hide show
  1. train/preflight.py +10 -2
train/preflight.py CHANGED
@@ -206,6 +206,13 @@ ounce100m_credentials.install()
206
  from huggingface_hub import HfApi
207
  api = HfApi(); tok = os.environ["HF_TOKEN"]
208
  repo = "Cion-lab/ounce100m-ckptbench-DELETEME"
 
 
 
 
 
 
 
209
  print("ensure_repo:", json.dumps(hubckpt.ensure_repo(repo, api, tok)), flush=True)
210
  d = "/kaggle/working/ckptbench/checkpoint-1"
211
  os.makedirs(d, exist_ok=True)
@@ -228,8 +235,9 @@ try:
228
  # second, independent proof: pull it back into a clean directory and compare hashes
229
  back = "/kaggle/working/ckptbench/restored"
230
  got = hubckpt.download_checkpoint(repo, "ckpt/checkpoint-1", back, api, token=tok)
231
- same = json.load(open(os.path.join(back, "cursor.json"))) == {"step": 1,
232
- "samples_consumed": 381500}
 
233
  # The repo is a timing rig, not an artifact: leaving a 1.7 GB public blob invites a future session
234
  # to mistake it for a checkpoint. Everything measurable is in the log by this point.
235
  api.delete_repo(repo_id=repo, repo_type="dataset", token=tok)
 
206
  from huggingface_hub import HfApi
207
  api = HfApi(); tok = os.environ["HF_TOKEN"]
208
  repo = "Cion-lab/ounce100m-ckptbench-DELETEME"
209
+ # Start from an empty repo: LFS dedupes identical bytes, so pushing into a repo that already holds this
210
+ # mock would time a no-op and report it as upload speed. Same lesson as the rehearsal's preclean.
211
+ try:
212
+ api.delete_repo(repo_id=repo, repo_type="dataset", token=tok)
213
+ print("preclean: deleted stale ckptbench repo", flush=True)
214
+ except Exception as e:
215
+ print("preclean:", type(e).__name__, str(e)[:100], flush=True)
216
  print("ensure_repo:", json.dumps(hubckpt.ensure_repo(repo, api, tok)), flush=True)
217
  d = "/kaggle/working/ckptbench/checkpoint-1"
218
  os.makedirs(d, exist_ok=True)
 
235
  # second, independent proof: pull it back into a clean directory and compare hashes
236
  back = "/kaggle/working/ckptbench/restored"
237
  got = hubckpt.download_checkpoint(repo, "ckpt/checkpoint-1", back, api, token=tok)
238
+ same = json.load(open(os.path.join(got["dir"], "cursor.json"))) == {
239
+ "step": 1, "samples_consumed": 381500}
240
+ print("download landed in", got["dir"], "files", got["files"], flush=True)
241
  # The repo is a timing rig, not an artifact: leaving a 1.7 GB public blob invites a future session
242
  # to mistake it for a checkpoint. Everything measurable is in the log by this point.
243
  api.delete_repo(repo_id=repo, repo_type="dataset", token=tok)