ViuAI commited on
Commit
b8da5cc
·
verified ·
1 Parent(s): 5366926

fix: auto-download tokenizer.json if LFS pointer detected

Browse files
Files changed (1) hide show
  1. model/scripts/train.py +42 -4
model/scripts/train.py CHANGED
@@ -495,6 +495,8 @@ def main():
495
  sample_every = C("sample_every", 1000)
496
  ckpt_dir = resolve(C("ckpt_dir", "../checkpoints"))
497
  log_dir = resolve(C("log_dir", "../../logs"))
 
 
498
  device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
499
  if args.device:
500
  device = torch.device(args.device)
@@ -536,10 +538,46 @@ def main():
536
 
537
  # tokenizer
538
  tok = None
539
- if Tokenizer is not None and tok_path is not None and Path(str(tok_path)).exists():
540
- tok = Tokenizer.from_file(str(tok_path))
541
- if R0:
542
- print(f"[tok] loaded {tok_path} vocab={tok.get_vocab_size()}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
543
  elif not args.smoke:
544
  if R0:
545
  print(f"[warn] tokenizer nahi mila ({tok_path}) — FINAL tokenizer Phase-2 me banega.")
 
495
  sample_every = C("sample_every", 1000)
496
  ckpt_dir = resolve(C("ckpt_dir", "../checkpoints"))
497
  log_dir = resolve(C("log_dir", "../../logs"))
498
+ hf_repo = C("hf_repo", "ViuAI/ViuMini-MoE-242M")
499
+ hf_token = os.environ.get("HF_TOKEN")
500
  device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
501
  if args.device:
502
  device = torch.device(args.device)
 
538
 
539
  # tokenizer
540
  tok = None
541
+ if Tokenizer is not None and tok_path is not None:
542
+ p_tok = Path(str(tok_path))
543
+ needs_download = False
544
+ if not p_tok.exists() or p_tok.stat().st_size < 10000:
545
+ needs_download = True
546
+ else:
547
+ try:
548
+ tok = Tokenizer.from_file(str(p_tok))
549
+ except Exception:
550
+ needs_download = True
551
+
552
+ if needs_download:
553
+ if R0:
554
+ print(f"[tok] {tok_path} missing or LFS pointer — auto-downloading valid tokenizer.json from Hugging Face...", flush=True)
555
+ from huggingface_hub import hf_hub_download
556
+ import shutil
557
+ try:
558
+ real_tok = hf_hub_download(
559
+ repo_id=hf_repo or "ViuAI/ViuMini-MoE-242M",
560
+ filename="tokenizer/outputs/tokenizer.json",
561
+ token=hf_token
562
+ )
563
+ p_tok.parent.mkdir(parents=True, exist_ok=True)
564
+ shutil.copy(real_tok, str(p_tok))
565
+ tok = Tokenizer.from_file(str(p_tok))
566
+ except Exception as e:
567
+ if R0:
568
+ print(f"[tok] fallback to dataset repo for tokenizer: {e}")
569
+ real_tok = hf_hub_download(
570
+ repo_id="ViuAI/viu-mini-raw-pretrain",
571
+ filename="tokenizer/tokenizer.json",
572
+ repo_type="dataset",
573
+ token=hf_token
574
+ )
575
+ p_tok.parent.mkdir(parents=True, exist_ok=True)
576
+ shutil.copy(real_tok, str(p_tok))
577
+ tok = Tokenizer.from_file(str(p_tok))
578
+
579
+ if tok is not None and R0:
580
+ print(f"[tok] loaded {p_tok} vocab={tok.get_vocab_size()}")
581
  elif not args.smoke:
582
  if R0:
583
  print(f"[warn] tokenizer nahi mila ({tok_path}) — FINAL tokenizer Phase-2 me banega.")