ViuAI commited on
Commit
46b3eaf
·
verified ·
1 Parent(s): 82b45f8

feat: non-blocking asynchronous checkpoint upload and local disk pruning

Browse files
Files changed (1) hide show
  1. model/scripts/train.py +143 -46
model/scripts/train.py CHANGED
@@ -9,11 +9,13 @@ Real:
9
  python train.py --config ../configs/train_config.yaml --model_config ../configs/viu1_moe_config.yaml
10
  """
11
  import argparse
 
12
  import contextlib
13
  import json
14
  import math
15
  import os
16
  import random
 
17
  import sys
18
  import time
19
  from pathlib import Path
@@ -273,6 +275,108 @@ def hf_push_ckpt(ckpt_dir, repo_id, token, path_in_repo="model/checkpoints", onl
273
  return False
274
 
275
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
276
  @torch.no_grad()
277
  def evaluate(model, ds, device, max_batches=20, use_amp=False, amp_dtype=torch.float16):
278
  model.eval()
@@ -834,6 +938,18 @@ def main():
834
  if R0:
835
  Path(ckpt_dir).mkdir(parents=True, exist_ok=True)
836
 
 
 
 
 
 
 
 
 
 
 
 
 
837
  model.train()
838
  t0 = time.time()
839
  step = start
@@ -936,52 +1052,30 @@ def main():
936
  logf.flush()
937
 
938
  if R0 and (not args.smoke) and step % save_every == 0:
939
- save_ckpt(Path(ckpt_dir) / f"step_{step}.pt", ckpt_model, opt, step, cfg, scaler=scaler)
940
- save_slim_ckpt(Path(ckpt_dir) / f"slim_step_{step}.pt", ckpt_model, cfg)
941
- print(f"[ckpt] saved step_{step}.pt + slim_step_{step}.pt")
942
- if do_hf_push and hf_every > 0 and (step // max(save_every, 1)) % hf_every == 0:
943
- hf_push_ckpt(ckpt_dir, hf_repo, hf_token, only_file=Path(ckpt_dir) / f"slim_step_{step}.pt",
944
- msg=f"ckpt step_{step}")
945
- # FULL ckpt HF push (resume-capable): har hf_push_full_every steps.
946
- # Kaggle session wipe-proof design: har push ke saath log file bhi HF pe
947
- # (logs/train.jsonl overwrite), aur purane full ckpt me se newest 2 hi rakho.
948
- if do_hf_push and hf_full_every > 0 and step % hf_full_every == 0:
949
- print(f"[hf-push-full] uploading step_{step}.pt (~GBs, time lagega)...", flush=True)
950
- ok = hf_push_ckpt(ckpt_dir, hf_repo, hf_token,
951
- only_file=Path(ckpt_dir) / f"step_{step}.pt",
952
- msg=f"full ckpt step_{step} (resume)")
953
- if R0:
954
- print(f"[hf-push-full] step_{step} {'OK' if ok else 'FAIL (local ckpt safe)'}", flush=True)
955
- if R0 and ok:
956
- try:
957
- from huggingface_hub import HfApi as _HfApi
958
- import re as _re
959
- _api = _HfApi(token=hf_token)
960
- # 1) live log HF pe (wipe-proof monitoring)
961
- try:
962
- if logf is not None:
963
- logf.flush()
964
- _logp = Path(log_dir) / "train.jsonl"
965
- if _logp.exists():
966
- _api.upload_file(path_or_fileobj=str(_logp), path_in_repo="logs/train.jsonl",
967
- repo_id=hf_repo, commit_message=f"log step_{step}")
968
- except Exception as e:
969
- print(f"[hf-push-full] WARN log upload: {str(e)[:120]}")
970
- # 2) repo bharne se roko: newest 2 full ckpt rakho, baaki delete
971
- try:
972
- _files = _api.list_repo_files(hf_repo, repo_type="model")
973
- def _s(p):
974
- m = _re.search(r"model/checkpoints/step_(\d+)\.pt$", p)
975
- return int(m.group(1)) if m else -1
976
- _fulls = sorted([f for f in _files if _s(f) >= 0], key=_s)
977
- for _old in _fulls[:-2]:
978
- _api.delete_file(path_in_repo=_old, repo_id=hf_repo,
979
- repo_type="model", commit_message=f"cleanup old {_old}")
980
- print(f"[hf-push-full] cleanup {_old}")
981
- except Exception as e:
982
- print(f"[hf-push-full] WARN cleanup: {str(e)[:120]}")
983
- except Exception as e:
984
- print(f"[hf-push-full] WARN post-push: {str(e)[:120]}")
985
 
986
  if R0 and (not args.smoke) and tok is not None and step % sample_every == 0:
987
  try:
@@ -996,6 +1090,9 @@ def main():
996
  if R0:
997
  print(f"[done] steps {total_done} time {fmt_hms(total_dt)} "
998
  f"avg_tok/s {tok_per_step * total_done / max(total_dt, 1e-6):,.0f}", flush=True)
 
 
 
999
  if R0 and do_hf_push:
1000
  save_slim_ckpt(Path(ckpt_dir) / f"slim_final_{step}.pt", ckpt_model, cfg)
1001
  hf_push_ckpt(ckpt_dir, hf_repo, hf_token, only_file=Path(ckpt_dir) / f"slim_final_{step}.pt",
 
9
  python train.py --config ../configs/train_config.yaml --model_config ../configs/viu1_moe_config.yaml
10
  """
11
  import argparse
12
+ import concurrent.futures
13
  import contextlib
14
  import json
15
  import math
16
  import os
17
  import random
18
+ import re
19
  import sys
20
  import time
21
  from pathlib import Path
 
275
  return False
276
 
277
 
278
+ class AsyncCheckpointManager:
279
+ """Non-blocking background checkpoint manager for Hugging Face upload and disk maintenance.
280
+ Ensures GPU training loop never pauses or blocks during remote checkpoint sync.
281
+ """
282
+ def __init__(self, repo_id, token, ckpt_dir=None, log_dir=None, enabled=True, keep_local_full=2, keep_remote_full=2):
283
+ self.repo_id = repo_id
284
+ self.token = token
285
+ self.ckpt_dir = Path(ckpt_dir) if ckpt_dir else None
286
+ self.log_dir = Path(log_dir) if log_dir else None
287
+ self.enabled = bool(enabled and repo_id and token)
288
+ self.keep_local_full = keep_local_full
289
+ self.keep_remote_full = keep_remote_full
290
+ self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=1, thread_name_prefix="hf_ckpt_uploader") if self.enabled else None
291
+ self.active_future = None
292
+
293
+ def prune_local_full_ckpts(self, current_step=None):
294
+ """Keep only newest N full checkpoints locally to prevent Kaggle 20GB disk overflow."""
295
+ if not self.ckpt_dir or not self.ckpt_dir.exists():
296
+ return
297
+ full_files = []
298
+ for f in self.ckpt_dir.glob("step_*.pt"):
299
+ m = re.match(r"^step_(\d+)\.pt$", f.name)
300
+ if m:
301
+ full_files.append((int(m.group(1)), f))
302
+ full_files.sort(key=lambda x: x[0])
303
+ if len(full_files) > self.keep_local_full:
304
+ for s, f in full_files[:-self.keep_local_full]:
305
+ try:
306
+ f.unlink()
307
+ print(f"[ckpt-local] pruned old local full ckpt: {f.name} (kept newest {self.keep_local_full})", flush=True)
308
+ except Exception as e:
309
+ print(f"[ckpt-local] WARN deleting {f.name}: {e}", flush=True)
310
+
311
+ def push_checkpoint_async(self, step, full_file=None, slim_file=None, log_file=None):
312
+ if not self.enabled or self.executor is None:
313
+ return None
314
+
315
+ def _worker():
316
+ t_start = time.time()
317
+ try:
318
+ from huggingface_hub import HfApi
319
+ api = HfApi(token=self.token)
320
+
321
+ # 1. Upload slim checkpoint if provided
322
+ if slim_file and Path(slim_file).exists():
323
+ p = Path(slim_file)
324
+ print(f"[hf-async] starting background upload: {p.name}...", flush=True)
325
+ api.upload_file(path_or_fileobj=str(p),
326
+ path_in_repo=f"model/checkpoints/{p.name}",
327
+ repo_id=self.repo_id,
328
+ commit_message=f"slim ckpt step_{step}")
329
+ print(f"[hf-async] {p.name} uploaded successfully", flush=True)
330
+
331
+ # 2. Upload full checkpoint if provided
332
+ if full_file and Path(full_file).exists():
333
+ p = Path(full_file)
334
+ print(f"[hf-async] starting background upload: {p.name} (~GBs)...", flush=True)
335
+ api.upload_file(path_or_fileobj=str(p),
336
+ path_in_repo=f"model/checkpoints/{p.name}",
337
+ repo_id=self.repo_id,
338
+ commit_message=f"full ckpt step_{step} (resume)")
339
+ print(f"[hf-async] {p.name} uploaded successfully", flush=True)
340
+
341
+ # 3. Clean up older full checkpoints on remote HF repo
342
+ try:
343
+ files = api.list_repo_files(self.repo_id, repo_type="model")
344
+ def _get_step(fn):
345
+ m = re.search(r"model/checkpoints/step_(\d+)\.pt$", fn)
346
+ return int(m.group(1)) if m else -1
347
+ full_ckpts = sorted([f for f in files if _get_step(f) >= 0], key=_get_step)
348
+ if len(full_ckpts) > self.keep_remote_full:
349
+ for old_ckpt in full_ckpts[:-self.keep_remote_full]:
350
+ api.delete_file(path_in_repo=old_ckpt, repo_id=self.repo_id,
351
+ repo_type="model", commit_message=f"prune old ckpt {old_ckpt}")
352
+ print(f"[hf-async] pruned remote old ckpt: {old_ckpt}", flush=True)
353
+ except Exception as e_prune:
354
+ print(f"[hf-async] WARN remote repo pruning: {str(e_prune)[:120]}", flush=True)
355
+
356
+ # 4. Upload live log file to HF for remote tracking
357
+ if log_file and Path(log_file).exists():
358
+ try:
359
+ api.upload_file(path_or_fileobj=str(log_file),
360
+ path_in_repo="logs/train.jsonl",
361
+ repo_id=self.repo_id,
362
+ commit_message=f"sync log step_{step}")
363
+ except Exception as e_log:
364
+ print(f"[hf-async] WARN log upload: {str(e_log)[:120]}", flush=True)
365
+
366
+ elapsed = time.time() - t_start
367
+ print(f"[hf-async] completed step {step} background sync in {elapsed:.1f}s (training was never paused)", flush=True)
368
+ except Exception as e:
369
+ print(f"[hf-async] WARN background upload step {step} failed: {str(e)[:200]} (training safe, local file intact)", flush=True)
370
+
371
+ self.active_future = self.executor.submit(_worker)
372
+ return self.active_future
373
+
374
+ def wait_pending(self, timeout=None):
375
+ """Wait for any active background upload to complete (e.g. before exiting training)."""
376
+ if self.executor is not None:
377
+ self.executor.shutdown(wait=True)
378
+
379
+
380
  @torch.no_grad()
381
  def evaluate(model, ds, device, max_batches=20, use_amp=False, amp_dtype=torch.float16):
382
  model.eval()
 
938
  if R0:
939
  Path(ckpt_dir).mkdir(parents=True, exist_ok=True)
940
 
941
+ async_uploader = None
942
+ if R0 and not args.smoke:
943
+ async_uploader = AsyncCheckpointManager(
944
+ repo_id=hf_repo,
945
+ token=hf_token,
946
+ ckpt_dir=ckpt_dir,
947
+ log_dir=log_dir,
948
+ enabled=do_hf_push,
949
+ keep_local_full=int(C("keep_local_full", 2)),
950
+ keep_remote_full=int(C("keep_remote_full", 2))
951
+ )
952
+
953
  model.train()
954
  t0 = time.time()
955
  step = start
 
1052
  logf.flush()
1053
 
1054
  if R0 and (not args.smoke) and step % save_every == 0:
1055
+ full_pt = Path(ckpt_dir) / f"step_{step}.pt"
1056
+ slim_pt = Path(ckpt_dir) / f"slim_step_{step}.pt"
1057
+ save_ckpt(full_pt, ckpt_model, opt, step, cfg, scaler=scaler)
1058
+ save_slim_ckpt(slim_pt, ckpt_model, cfg)
1059
+ print(f"[ckpt] saved step_{step}.pt + slim_step_{step}.pt to disk")
1060
+
1061
+ # Prune older local checkpoints to ensure Kaggle disk never fills up
1062
+ if async_uploader is not None:
1063
+ async_uploader.prune_local_full_ckpts(step)
1064
+
1065
+ # Asynchronous background upload to Hugging Face Hub (zero GPU blocking)
1066
+ push_slim = (do_hf_push and hf_every > 0 and (step // max(save_every, 1)) % hf_every == 0)
1067
+ push_full = (do_hf_push and hf_full_every > 0 and step % hf_full_every == 0)
1068
+ if push_slim or push_full:
1069
+ if logf is not None:
1070
+ logf.flush()
1071
+ if async_uploader is not None and async_uploader.enabled:
1072
+ print(f"[ckpt] dispatching background HF upload for step {step} (GPU training continues immediately)...", flush=True)
1073
+ async_uploader.push_checkpoint_async(
1074
+ step=step,
1075
+ full_file=full_pt if push_full else None,
1076
+ slim_file=slim_pt if push_slim else None,
1077
+ log_file=Path(log_dir) / "train.jsonl"
1078
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1079
 
1080
  if R0 and (not args.smoke) and tok is not None and step % sample_every == 0:
1081
  try:
 
1090
  if R0:
1091
  print(f"[done] steps {total_done} time {fmt_hms(total_dt)} "
1092
  f"avg_tok/s {tok_per_step * total_done / max(total_dt, 1e-6):,.0f}", flush=True)
1093
+ if R0 and async_uploader is not None and async_uploader.enabled:
1094
+ print("[hf-async] waiting for any in-flight background checkpoint uploads to complete...", flush=True)
1095
+ async_uploader.wait_pending()
1096
  if R0 and do_hf_push:
1097
  save_slim_ckpt(Path(ckpt_dir) / f"slim_final_{step}.pt", ckpt_model, cfg)
1098
  hf_push_ckpt(ckpt_dir, hf_repo, hf_token, only_file=Path(ckpt_dir) / f"slim_final_{step}.pt",