fix: auto-download tokenizer.json if LFS pointer detected
Browse files- model/scripts/train.py +42 -4
model/scripts/train.py
CHANGED
|
@@ -495,6 +495,8 @@ def main():
|
|
| 495 |
sample_every = C("sample_every", 1000)
|
| 496 |
ckpt_dir = resolve(C("ckpt_dir", "../checkpoints"))
|
| 497 |
log_dir = resolve(C("log_dir", "../../logs"))
|
|
|
|
|
|
|
| 498 |
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 499 |
if args.device:
|
| 500 |
device = torch.device(args.device)
|
|
@@ -536,10 +538,46 @@ def main():
|
|
| 536 |
|
| 537 |
# tokenizer
|
| 538 |
tok = None
|
| 539 |
-
if Tokenizer is not None and tok_path is not None
|
| 540 |
-
|
| 541 |
-
|
| 542 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 543 |
elif not args.smoke:
|
| 544 |
if R0:
|
| 545 |
print(f"[warn] tokenizer nahi mila ({tok_path}) — FINAL tokenizer Phase-2 me banega.")
|
|
|
|
| 495 |
sample_every = C("sample_every", 1000)
|
| 496 |
ckpt_dir = resolve(C("ckpt_dir", "../checkpoints"))
|
| 497 |
log_dir = resolve(C("log_dir", "../../logs"))
|
| 498 |
+
hf_repo = C("hf_repo", "ViuAI/ViuMini-MoE-242M")
|
| 499 |
+
hf_token = os.environ.get("HF_TOKEN")
|
| 500 |
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 501 |
if args.device:
|
| 502 |
device = torch.device(args.device)
|
|
|
|
| 538 |
|
| 539 |
# tokenizer
|
| 540 |
tok = None
|
| 541 |
+
if Tokenizer is not None and tok_path is not None:
|
| 542 |
+
p_tok = Path(str(tok_path))
|
| 543 |
+
needs_download = False
|
| 544 |
+
if not p_tok.exists() or p_tok.stat().st_size < 10000:
|
| 545 |
+
needs_download = True
|
| 546 |
+
else:
|
| 547 |
+
try:
|
| 548 |
+
tok = Tokenizer.from_file(str(p_tok))
|
| 549 |
+
except Exception:
|
| 550 |
+
needs_download = True
|
| 551 |
+
|
| 552 |
+
if needs_download:
|
| 553 |
+
if R0:
|
| 554 |
+
print(f"[tok] {tok_path} missing or LFS pointer — auto-downloading valid tokenizer.json from Hugging Face...", flush=True)
|
| 555 |
+
from huggingface_hub import hf_hub_download
|
| 556 |
+
import shutil
|
| 557 |
+
try:
|
| 558 |
+
real_tok = hf_hub_download(
|
| 559 |
+
repo_id=hf_repo or "ViuAI/ViuMini-MoE-242M",
|
| 560 |
+
filename="tokenizer/outputs/tokenizer.json",
|
| 561 |
+
token=hf_token
|
| 562 |
+
)
|
| 563 |
+
p_tok.parent.mkdir(parents=True, exist_ok=True)
|
| 564 |
+
shutil.copy(real_tok, str(p_tok))
|
| 565 |
+
tok = Tokenizer.from_file(str(p_tok))
|
| 566 |
+
except Exception as e:
|
| 567 |
+
if R0:
|
| 568 |
+
print(f"[tok] fallback to dataset repo for tokenizer: {e}")
|
| 569 |
+
real_tok = hf_hub_download(
|
| 570 |
+
repo_id="ViuAI/viu-mini-raw-pretrain",
|
| 571 |
+
filename="tokenizer/tokenizer.json",
|
| 572 |
+
repo_type="dataset",
|
| 573 |
+
token=hf_token
|
| 574 |
+
)
|
| 575 |
+
p_tok.parent.mkdir(parents=True, exist_ok=True)
|
| 576 |
+
shutil.copy(real_tok, str(p_tok))
|
| 577 |
+
tok = Tokenizer.from_file(str(p_tok))
|
| 578 |
+
|
| 579 |
+
if tok is not None and R0:
|
| 580 |
+
print(f"[tok] loaded {p_tok} vocab={tok.get_vocab_size()}")
|
| 581 |
elif not args.smoke:
|
| 582 |
if R0:
|
| 583 |
print(f"[warn] tokenizer nahi mila ({tok_path}) — FINAL tokenizer Phase-2 me banega.")
|