Download code/train.py from ViuAI/ViuRec: direct link, hf CLI and curl.
- Browser
- Download file 15.6 kB
-
https://huggingface.co/ViuAI/ViuRec/resolve/fb73e4792f1bfa00deb74571a9f972ecdfa0d0a4/code/train.py
- Command line
-
hf download hf://ViuAI/ViuRec@fb73e4792f1bfa00deb74571a9f972ecdfa0d0a4/code/train.py
-
curl -L -o train.py https://huggingface.co/ViuAI/ViuRec/resolve/fb73e4792f1bfa00deb74571a9f972ecdfa0d0a4/code/train.py
15.6 kB
| import os | |
| # Setup env vars for robust Kaggle training | |
| os.environ["NCCL_P2P_DISABLE"] = "1" | |
| os.environ["NCCL_IB_DISABLE"] = "1" | |
| os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1" | |
| os.environ["HF_HUB_DISABLE_PROGRESS_BARS"] = "1" | |
| import math | |
| import time | |
| import argparse | |
| import logging | |
| from pathlib import Path | |
| import torch | |
| import torch.distributed as dist | |
| from torch.nn.parallel import DistributedDataParallel as DDP | |
| from torch.cuda.amp import GradScaler | |
| import numpy as np | |
| from transformers import AutoTokenizer | |
| from huggingface_hub import hf_hub_download, HfApi | |
| from huggingface_hub.utils import EntryNotFoundError, HfHubHTTPError | |
| try: | |
| from kaggle_secrets import UserSecretsClient | |
| HAS_KAGGLE_SECRETS = True | |
| except ImportError: | |
| HAS_KAGGLE_SECRETS = False | |
| logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") | |
| logger = logging.getLogger(__name__) | |
| def get_hf_token(): | |
| token = os.environ.get("HF_TOKEN") | |
| if token: | |
| logger.info("Loaded HF_TOKEN from environment.") | |
| return token | |
| if HAS_KAGGLE_SECRETS: | |
| try: | |
| token = UserSecretsClient().get_secret("HF_TOKEN") | |
| if token: | |
| os.environ["HF_TOKEN"] = token | |
| logger.info("Loaded HF_TOKEN from kaggle_secrets.") | |
| return token | |
| except Exception as e: | |
| logger.warning(f"Failed to get HF_TOKEN from kaggle_secrets: {e}") | |
| raise ValueError("HF_TOKEN not found in environment or kaggle_secrets.") | |
| def setup_ddp(): | |
| if "RANK" not in os.environ or "WORLD_SIZE" not in os.environ: | |
| return 0, 0, 1, torch.device("cuda" if torch.cuda.is_available() else "cpu"), False | |
| dist.init_process_group("nccl") | |
| local_rank = int(os.environ["LOCAL_RANK"]) | |
| device = torch.device(f"cuda:{local_rank}") | |
| torch.cuda.set_device(device) | |
| return int(os.environ["RANK"]), local_rank, int(os.environ["WORLD_SIZE"]), device, True | |
| def load_tokenizer_robust(hf_token: str, model_repo: str, subfolder: str, is_main: bool): | |
| if is_main: logger.info("Loading tokenizer...") | |
| local_candidates = [ | |
| "/kaggle/working/ViuRec/tokenizer", | |
| "/kaggle/input/viuai-500m-tokenizer", | |
| "./tokenizer", | |
| "./viuai-500m-tokenizer" | |
| ] | |
| for fast in [True, False]: | |
| for p_str in local_candidates: | |
| p = Path(p_str) | |
| if (p / "tokenizer.json").exists() or (p / "vocab.json").exists(): | |
| try: | |
| return AutoTokenizer.from_pretrained(str(p), local_files_only=True, use_fast=fast, trust_remote_code=True) | |
| except Exception as e: | |
| if "sentencepiece" in str(e).lower() or "tiktoken" in str(e).lower(): | |
| continue | |
| try: | |
| return AutoTokenizer.from_pretrained(model_repo, subfolder=subfolder, token=hf_token, use_fast=fast, trust_remote_code=True) | |
| except Exception as e: | |
| pass | |
| try: | |
| return AutoTokenizer.from_pretrained(model_repo, token=hf_token, use_fast=fast, trust_remote_code=True) | |
| except Exception as e: | |
| pass | |
| if is_main: | |
| logger.warning("Tokenizer failed, using dummy vocab 64000") | |
| return None | |
| def load_checkpoint(model, optimizer, scaler, api, token, repo_id, filename="checkpoints/latest_chk.pt", strict=False): | |
| local_path = "latest_chk.pt" | |
| try: | |
| logger.info(f"Checking for existing checkpoint on HF: {filename}") | |
| hf_hub_download(repo_id=repo_id, filename=filename, local_dir=".", token=token) | |
| # Security Note: weights_only=False is used here to allow loading the optimizer state safely inside our trusted environment. | |
| ckpt = torch.load(local_path, map_location="cpu", weights_only=False) | |
| missing, unexpected = model.load_state_dict(ckpt["model"], strict=strict) | |
| if missing or unexpected: | |
| logger.warning(f"Checkpoint loaded with Missing keys: {missing} | Unexpected keys: {unexpected}") | |
| else: | |
| logger.info("Model weights loaded successfully.") | |
| optimizer.load_state_dict(ckpt["optimizer"]) | |
| if "scaler" in ckpt and scaler is not None: | |
| scaler.load_state_dict(ckpt["scaler"]) | |
| step = ckpt.get("step", 0) | |
| logger.info(f"Resuming from step {step}") | |
| Path(local_path).unlink(missing_ok=True) | |
| return step | |
| except (EntryNotFoundError, HfHubHTTPError): | |
| logger.info("No checkpoint found on HF. Starting from scratch.") | |
| return 0 | |
| except Exception as e: | |
| logger.error(f"Error loading checkpoint: {e}") | |
| return 0 | |
| class ShardPool: | |
| def __init__(self, api, repo_id, work_dir, token, vocab_size, max_pool=3): | |
| self.api = api | |
| self.repo_id = repo_id | |
| self.work_dir = Path(work_dir) | |
| self.work_dir.mkdir(parents=True, exist_ok=True) | |
| self.token = token | |
| self.max_pool = max_pool | |
| self.pool = [] | |
| self.available_files = [] | |
| self.vocab_size = vocab_size | |
| self._refresh_file_list() | |
| def _refresh_file_list(self): | |
| files = self.api.list_repo_files(repo_id=self.repo_id, repo_type="dataset", token=self.token) | |
| self.available_files = [f for f in files if f.startswith("shards/hi_indiccorp_") and f.endswith(".bin")] | |
| np.random.shuffle(self.available_files) | |
| def fill(self, ctx_len): | |
| max_attempts = self.max_pool * 5 | |
| attempts = 0 | |
| while len(self.pool) < self.max_pool: | |
| if attempts >= max_attempts: | |
| raise RuntimeError("Failed to fill ShardPool: too many invalid shards.") | |
| if not self.available_files: | |
| self._refresh_file_list() | |
| f = self.available_files.pop() | |
| local_p = hf_hub_download(repo_id=self.repo_id, repo_type="dataset", filename=f, local_dir=self.work_dir, token=self.token) | |
| # Safe dtype selection | |
| dtype = np.uint16 if self.vocab_size <= 65535 else np.uint32 | |
| mmap_arr = np.memmap(local_p, dtype=dtype, mode="r") | |
| if len(mmap_arr) < ctx_len + 1: | |
| logger.warning(f"Shard {f} is too small ({len(mmap_arr)} tokens). Discarding.") | |
| del mmap_arr | |
| Path(local_p).unlink(missing_ok=True) | |
| attempts += 1 | |
| continue | |
| self.pool.append((local_p, mmap_arr)) | |
| logger.info(f"Loaded shard {f}") | |
| def sample_batch(self, batch_size, ctx_len, rng): | |
| xs, ys = [], [] | |
| while len(xs) < batch_size: | |
| idx = rng.integers(0, len(self.pool)) | |
| _, tokens = self.pool[idx] | |
| max_start = len(tokens) - ctx_len - 1 | |
| if max_start <= 0: | |
| continue | |
| i = rng.integers(0, max_start) | |
| xs.append(np.array(tokens[i:i+ctx_len], dtype=np.int64)) | |
| ys.append(np.array(tokens[i+1:i+1+ctx_len], dtype=np.int64)) | |
| return np.stack(xs), np.stack(ys) | |
| def rotate_one(self): | |
| if not self.pool: return | |
| old_path, old_mmap = self.pool.pop(0) | |
| del old_mmap | |
| try: | |
| Path(old_path).unlink(missing_ok=True) | |
| logger.info(f"Rotated out shard {Path(old_path).name}") | |
| except Exception as e: | |
| logger.warning(f"Failed to delete {old_path}: {e}") | |
| def main(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--max-steps", type=int, default=100) | |
| parser.add_argument("--micro-batch", type=int, default=2) | |
| parser.add_argument("--grad-accum", type=int, default=16) | |
| parser.add_argument("--eval-every", type=int, default=500) | |
| parser.add_argument("--save-every", type=int, default=500) | |
| parser.add_argument("--rotate-every", type=int, default=500) | |
| parser.add_argument("--lr", type=float, default=3e-4) | |
| parser.add_argument("--weight-decay", type=float, default=0.1) | |
| parser.add_argument("--ctx-len", type=int, default=2048) | |
| parser.add_argument("--vocab-size", type=int, default=None) | |
| args = parser.parse_args() | |
| token = get_hf_token() | |
| rank, local_rank, world_size, device, is_ddp = setup_ddp() | |
| is_main = (rank == 0) | |
| # Random seed | |
| seed = 42 + rank | |
| torch.manual_seed(seed) | |
| np.random.seed(seed) | |
| rng = np.random.default_rng(seed) | |
| # Load resources | |
| tokenizer = load_tokenizer_robust(token, "ViuAI/ViuRec", "tokenizer", is_main) | |
| vocab_size = args.vocab_size or (len(tokenizer) if tokenizer is not None else 64000) | |
| from model import ViuResonance100M, ModelConfig | |
| config = ModelConfig(vocab_size=vocab_size) | |
| model = ViuResonance100M(config).to(device) | |
| if is_ddp: | |
| model = DDP(model, device_ids=[local_rank], output_device=local_rank, broadcast_buffers=False) | |
| unwrapped_model = model.module | |
| else: | |
| unwrapped_model = model | |
| # Optional torch.compile fallback | |
| try: | |
| model = torch.compile(model) | |
| if is_main: logger.info("Compiled model successfully.") | |
| except Exception as e: | |
| if is_main: logger.warning(f"Failed to compile model: {e}. Falling back to eager mode.") | |
| # Optimizer | |
| try: | |
| optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=args.weight_decay, betas=(0.9, 0.95), fused=True) | |
| except: | |
| if is_main: logger.warning("fused=True failed for AdamW. Falling back to non-fused.") | |
| optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=args.weight_decay, betas=(0.9, 0.95)) | |
| # GradScaler for safe fp16/bf16 | |
| scaler = GradScaler(enabled=not torch.cuda.is_bf16_supported()) | |
| api = HfApi() | |
| start_step = load_checkpoint(unwrapped_model, optimizer, scaler, api, token, "ViuAI/ViuRec", strict=False) | |
| work_dir = Path("/kaggle/working" if Path("/kaggle/working").exists() else ".") | |
| shard_pool = ShardPool(api, "ViuAI/ViuCorp", work_dir / "shards", token, vocab_size) | |
| shard_pool.fill(args.ctx_len) | |
| # Val Data Safely (cap at 20M tokens) | |
| val_tokens = None | |
| if is_main: | |
| logger.info("Loading validation data...") | |
| try: | |
| val_files = [f for f in api.list_repo_files(repo_id="ViuAI/ViuCorp", repo_type="dataset", token=token) if f.startswith("val_shards/")] | |
| if val_files: | |
| val_p = hf_hub_download(repo_id="ViuAI/ViuCorp", repo_type="dataset", filename=val_files[0], local_dir=work_dir, token=token) | |
| dtype = np.uint16 if vocab_size <= 65535 else np.uint32 | |
| val_arr = np.memmap(val_p, dtype=dtype, mode="r") | |
| max_toks = 20_000_000 | |
| val_tokens = np.array(val_arr[:max_toks]) # Copy to RAM, safely capped | |
| del val_arr | |
| Path(val_p).unlink(missing_ok=True) | |
| except Exception as e: | |
| logger.warning(f"Failed to load val data: {e}") | |
| def evaluate(): | |
| if val_tokens is None or len(val_tokens) < args.ctx_len + 1: return 0.0 | |
| unwrapped_model.eval() | |
| val_loss = 0.0 | |
| val_iters = 20 | |
| for _ in range(val_iters): | |
| i = rng.integers(0, len(val_tokens) - args.ctx_len - 1) | |
| x = torch.from_numpy(val_tokens[i:i+args.ctx_len].astype(np.int64)).unsqueeze(0).to(device, non_blocking=True) | |
| y = torch.from_numpy(val_tokens[i+1:i+1+args.ctx_len].astype(np.int64)).unsqueeze(0).to(device, non_blocking=True) | |
| with torch.autocast(device_type="cuda", dtype=torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16): | |
| # Pass use_checkpoint=False for evaluation | |
| _, loss = unwrapped_model(x, y, use_checkpoint=False) | |
| val_loss += loss.item() | |
| unwrapped_model.train() | |
| return val_loss / val_iters | |
| # Training Loop | |
| model.train() | |
| if is_main: | |
| logger.info(f"Starting training for {args.max_steps} steps...") | |
| for step in range(start_step, args.max_steps): | |
| # LR Schedule | |
| warmup = 1000 | |
| progress = min(1.0, step / max(1, warmup)) | |
| lr = args.lr * progress if step < warmup else args.lr * (0.5 * (1.0 + math.cos(math.pi * (step - warmup) / max(1, args.max_steps - warmup)))) | |
| for param_group in optimizer.param_groups: | |
| param_group['lr'] = lr | |
| optimizer.zero_grad(set_to_none=True) | |
| accum_loss = 0.0 | |
| t0 = time.time() | |
| for micro_i in range(args.grad_accum): | |
| x_np, y_np = shard_pool.sample_batch(args.micro_batch, args.ctx_len, rng) | |
| x = torch.from_numpy(x_np).to(device, non_blocking=True) | |
| y = torch.from_numpy(y_np).to(device, non_blocking=True) | |
| with torch.autocast(device_type="cuda", dtype=torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16): | |
| _, loss = model(x, y) | |
| loss = loss / args.grad_accum | |
| scaler.scale(loss).backward() | |
| accum_loss += loss.item() | |
| del x, y, loss | |
| # Safe Gradient Clipping with Scaler | |
| scaler.unscale_(optimizer) | |
| torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) | |
| # Check for NaNs | |
| has_nan = not math.isfinite(accum_loss) | |
| for p in model.parameters(): | |
| if p.grad is not None and not torch.isfinite(p.grad).all(): | |
| has_nan = True | |
| break | |
| if has_nan: | |
| if is_main: logger.warning(f"NaN/Inf detected at step {step}! Skipping optimizer step.") | |
| optimizer.zero_grad(set_to_none=True) | |
| continue | |
| scaler.step(optimizer) | |
| scaler.update() | |
| dt = time.time() - t0 | |
| if is_main and step % 10 == 0: | |
| tok_s = (args.micro_batch * args.grad_accum * args.ctx_len * world_size) / dt if dt > 0 else 0 | |
| mem_alloc = torch.cuda.memory_allocated(device) / 1e9 | |
| mem_res = torch.cuda.memory_reserved(device) / 1e9 | |
| logger.info(f"step {step:5d}/{args.max_steps} | loss {accum_loss:.4f} | lr {lr:.2e} | {tok_s:.0f} tok/s | mem alloc={mem_alloc:.2f}GB res={mem_res:.2f}GB") | |
| if step > start_step and step % args.rotate_every == 0: | |
| shard_pool.rotate_one() | |
| shard_pool.fill(args.ctx_len) | |
| if step > start_step and step % args.eval_every == 0 and is_main: | |
| val_loss = evaluate() | |
| logger.info(f"--- Eval step {step}: val_loss {val_loss:.4f} ---") | |
| if step > start_step and step % args.save_every == 0: | |
| if is_ddp: dist.barrier() | |
| if is_main: | |
| logger.info(f"Saving checkpoint at step {step}...") | |
| chk = { | |
| "model": unwrapped_model.state_dict(), | |
| "optimizer": optimizer.state_dict(), | |
| "scaler": scaler.state_dict(), | |
| "step": step, | |
| } | |
| torch.save(chk, "latest_chk.pt") | |
| try: | |
| api.upload_file( | |
| path_or_fileobj="latest_chk.pt", | |
| path_in_repo="checkpoints/latest_chk.pt", | |
| repo_id="ViuAI/ViuRec", | |
| token=token | |
| ) | |
| logger.info("Upload successful.") | |
| except Exception as e: | |
| logger.error(f"Upload failed: {e}") | |
| if is_ddp: dist.barrier() | |
| if is_ddp: | |
| dist.destroy_process_group() | |
| if __name__ == "__main__": | |
| main() | |