ViuAI commited on
Commit
7c25c00
·
verified ·
1 Parent(s): 3bb9733

Upload 2 files

Browse files
Files changed (2) hide show
  1. code/model.py +75 -35
  2. code/train.py +325 -345
code/model.py CHANGED
@@ -3,29 +3,47 @@ import torch
3
  import torch.nn as nn
4
  import torch.nn.functional as F
5
  from torch.utils.checkpoint import checkpoint
 
 
 
 
 
 
 
 
 
 
 
6
 
7
  class RMSNorm(nn.Module):
8
- def __init__(self, dim, eps=1e-5):
9
  super().__init__()
10
- self.w = nn.Parameter(torch.ones(dim))
11
  self.eps = eps
12
 
13
- def forward(self, x):
14
- return (x.float() * torch.rsqrt(x.float().pow(2).mean(-1, keepdim=True) + self.eps) * self.w).type_as(x)
 
 
15
 
16
  class ResonanceLayerKaggle(nn.Module):
17
- def __init__(self, d_model=1024, n_heads=16):
18
  super().__init__()
19
- self.n_heads = n_heads
20
- self.head_dim = d_model // n_heads
21
- self.to_amp = nn.Linear(d_model, d_model, bias=False)
22
- self.to_freq = nn.Linear(d_model, n_heads, bias=False)
23
- self.to_phase = nn.Linear(d_model, n_heads, bias=False)
24
- self.to_v = nn.Linear(d_model, d_model, bias=False)
25
- self.to_out = nn.Linear(d_model, d_model, bias=False)
26
- self.norm = RMSNorm(d_model)
 
 
 
 
 
27
 
28
- def forward(self, x):
29
  B, T, C = x.shape
30
  h = self.norm(x)
31
  A = self.to_amp(h).view(B, T, self.n_heads, self.head_dim)
@@ -33,33 +51,43 @@ class ResonanceLayerKaggle(nn.Module):
33
  freq = torch.tanh(self.to_freq(h)) * 2.0 # B,T,H
34
  phase = torch.tanh(self.to_phase(h)) * math.pi
35
 
36
- # Distance - causal ke liye
37
- pos = torch.arange(T, device=x.device, dtype=x.dtype)
38
- chunk = 512 # safe for Kaggle T4
39
  out = torch.zeros(B, T, C, device=x.device, dtype=x.dtype)
40
 
41
- for i in range(0, T, chunk):
42
- end = min(i+chunk, T)
43
  phase_i = phase[:, i:end, :].permute(0,2,1).unsqueeze(-1) # B,H,chunk,1
44
  phase_j = phase.permute(0,2,1).unsqueeze(-2) # B,H,1,T
45
 
46
- dist = (torch.arange(i, end, device=x.device)[:,None] - pos[None,:]).abs().clamp(min=0)
47
- dist = dist.view(1, 1, end-i, T)
 
 
 
48
 
49
  freq_i = freq[:, i:end, :].permute(0,2,1).unsqueeze(-1)
50
  angle = (phase_i - phase_j) + freq_i * dist * 0.05
51
- score = torch.cos(angle) / math.sqrt(self.head_dim) # Normalize to prevent explosion
52
 
53
- causal_mask = (torch.arange(i, end, device=x.device)[:,None] >= pos[None,:]).to(x.dtype).view(1,1,end-i,T)
54
- score = score * causal_mask
 
 
 
 
 
 
 
55
 
56
  amp = A.norm(dim=-1).permute(0,2,1) # B,H,T
57
  amp_i = amp[:,:,i:end].unsqueeze(-1)
58
  amp_j = amp.unsqueeze(-2)
59
  score = score * torch.sigmoid(amp_i * amp_j)
60
 
61
- V_t = V.permute(0,2,1,3) # B,H,T,D
62
  Bh = B * self.n_heads
 
 
63
  s = score.reshape(Bh, end-i, T)
64
  v = V_t.reshape(Bh, T, self.head_dim)
65
  o = torch.bmm(s, v).view(B, self.n_heads, end-i, self.head_dim).permute(0,2,1,3).reshape(B, end-i, C)
@@ -68,28 +96,40 @@ class ResonanceLayerKaggle(nn.Module):
68
  return self.to_out(out)
69
 
70
  class ViuResonance100M(nn.Module):
71
- def __init__(self, vocab_size=64000, d_model=1024, n_layers=12, n_heads=16, dropout=0.1):
72
  super().__init__()
73
- self.emb = nn.Embedding(vocab_size, d_model)
74
- self.layers = nn.ModuleList([ResonanceLayerKaggle(d_model, n_heads) for _ in range(n_layers)])
75
- self.norm = RMSNorm(d_model)
76
- self.head = nn.Linear(d_model, vocab_size, bias=False)
77
- self.head.weight = self.emb.weight
78
- self.resid_dropout = nn.Dropout(dropout)
 
 
 
79
  self.apply(self._init_weights)
 
 
80
 
81
- def _init_weights(self, module):
82
  if isinstance(module, nn.Linear):
83
  torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
84
  elif isinstance(module, nn.Embedding):
85
  torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
 
 
86
 
87
- def forward(self, idx, targets=None):
88
  x = self.emb(idx)
89
  for layer in self.layers:
90
- x = x + self.resid_dropout(checkpoint(layer, x, use_reentrant=False))
 
 
 
 
91
  logits = self.head(self.norm(x))
92
  loss = None
93
  if targets is not None:
 
94
  loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-100)
95
  return logits, loss
 
3
  import torch.nn as nn
4
  import torch.nn.functional as F
5
  from torch.utils.checkpoint import checkpoint
6
+ from dataclasses import dataclass
7
+
8
+ @dataclass
9
+ class ModelConfig:
10
+ """Configuration class for ViuResonance100M."""
11
+ vocab_size: int = 64000
12
+ d_model: int = 1024
13
+ n_layers: int = 12
14
+ n_heads: int = 16
15
+ dropout: float = 0.1
16
+ chunk_size: int = 512
17
 
18
  class RMSNorm(nn.Module):
19
+ def __init__(self, dim: int, eps: float = 1e-6):
20
  super().__init__()
21
+ self.weight = nn.Parameter(torch.ones(dim))
22
  self.eps = eps
23
 
24
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
25
+ # Enforce float32 accumulation for stability, then cast back to original dtype
26
+ norm_x = x.float() * torch.rsqrt(x.float().pow(2).mean(-1, keepdim=True) + self.eps) * self.weight.float()
27
+ return norm_x.to(x.dtype)
28
 
29
  class ResonanceLayerKaggle(nn.Module):
30
+ def __init__(self, config: ModelConfig):
31
  super().__init__()
32
+ if config.d_model % config.n_heads != 0:
33
+ raise ValueError(f"d_model ({config.d_model}) must be divisible by n_heads ({config.n_heads})")
34
+
35
+ self.n_heads = config.n_heads
36
+ self.head_dim = config.d_model // config.n_heads
37
+ self.chunk_size = config.chunk_size
38
+
39
+ self.to_amp = nn.Linear(config.d_model, config.d_model, bias=False)
40
+ self.to_freq = nn.Linear(config.d_model, config.n_heads, bias=False)
41
+ self.to_phase = nn.Linear(config.d_model, config.n_heads, bias=False)
42
+ self.to_v = nn.Linear(config.d_model, config.d_model, bias=False)
43
+ self.to_out = nn.Linear(config.d_model, config.d_model, bias=False)
44
+ self.norm = RMSNorm(config.d_model)
45
 
46
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
47
  B, T, C = x.shape
48
  h = self.norm(x)
49
  A = self.to_amp(h).view(B, T, self.n_heads, self.head_dim)
 
51
  freq = torch.tanh(self.to_freq(h)) * 2.0 # B,T,H
52
  phase = torch.tanh(self.to_phase(h)) * math.pi
53
 
54
+ # Use torch.long for absolute positions to avoid float precision issues during subtraction
55
+ pos = torch.arange(T, device=x.device, dtype=torch.long)
 
56
  out = torch.zeros(B, T, C, device=x.device, dtype=x.dtype)
57
 
58
+ for i in range(0, T, self.chunk_size):
59
+ end = min(i+self.chunk_size, T)
60
  phase_i = phase[:, i:end, :].permute(0,2,1).unsqueeze(-1) # B,H,chunk,1
61
  phase_j = phase.permute(0,2,1).unsqueeze(-2) # B,H,1,T
62
 
63
+ # Robust distance calculation in torch.long to prevent overflow/dtype issues
64
+ row_idx = torch.arange(i, end, device=x.device, dtype=torch.long)[:, None]
65
+ col_idx = pos[None, :]
66
+ dist_long = (row_idx - col_idx).clamp(min=0)
67
+ dist = dist_long.to(freq.dtype).view(1, 1, end-i, T)
68
 
69
  freq_i = freq[:, i:end, :].permute(0,2,1).unsqueeze(-1)
70
  angle = (phase_i - phase_j) + freq_i * dist * 0.05
 
71
 
72
+ # Pure Cosine similarity. We apply the sqrt(d) scaling during the gating phase below
73
+ score = torch.cos(angle)
74
+
75
+ # Boolean causal mask for precision and stability
76
+ causal_bool = (row_idx >= col_idx).view(1,1,end-i,T)
77
+
78
+ # Mask out future tokens
79
+ score = score.masked_fill(~causal_bool, 0.0)
80
+ score = score / math.sqrt(self.head_dim)
81
 
82
  amp = A.norm(dim=-1).permute(0,2,1) # B,H,T
83
  amp_i = amp[:,:,i:end].unsqueeze(-1)
84
  amp_j = amp.unsqueeze(-2)
85
  score = score * torch.sigmoid(amp_i * amp_j)
86
 
87
+ V_t = V.permute(0,2,1,3).contiguous() # B,H,T,D
88
  Bh = B * self.n_heads
89
+ assert C == self.n_heads * self.head_dim, f"C ({C}) != n_heads*head_dim ({self.n_heads*self.head_dim})"
90
+
91
  s = score.reshape(Bh, end-i, T)
92
  v = V_t.reshape(Bh, T, self.head_dim)
93
  o = torch.bmm(s, v).view(B, self.n_heads, end-i, self.head_dim).permute(0,2,1,3).reshape(B, end-i, C)
 
96
  return self.to_out(out)
97
 
98
  class ViuResonance100M(nn.Module):
99
+ def __init__(self, config: ModelConfig):
100
  super().__init__()
101
+ self.config = config
102
+ self.emb = nn.Embedding(config.vocab_size, config.d_model)
103
+ self.layers = nn.ModuleList([ResonanceLayerKaggle(config) for _ in range(config.n_layers)])
104
+ self.norm = RMSNorm(config.d_model)
105
+ self.head = nn.Linear(config.d_model, config.vocab_size, bias=False)
106
+ self.resid_dropout = nn.Dropout(config.dropout)
107
+
108
+ # Proper initialization sequence to preserve weight tying!
109
+ # First randomly initialize everything...
110
  self.apply(self._init_weights)
111
+ # THEN explicitly tie the weights
112
+ self.head.weight = self.emb.weight
113
 
114
+ def _init_weights(self, module: nn.Module):
115
  if isinstance(module, nn.Linear):
116
  torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
117
  elif isinstance(module, nn.Embedding):
118
  torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
119
+ elif isinstance(module, RMSNorm):
120
+ torch.nn.init.ones_(module.weight)
121
 
122
+ def forward(self, idx: torch.Tensor, targets: torch.Tensor = None, use_checkpoint: bool = True) -> tuple[torch.Tensor, torch.Tensor | None]:
123
  x = self.emb(idx)
124
  for layer in self.layers:
125
+ if self.training and use_checkpoint:
126
+ x = x + self.resid_dropout(checkpoint(layer, x, use_reentrant=False))
127
+ else:
128
+ x = x + self.resid_dropout(layer(x))
129
+
130
  logits = self.head(self.norm(x))
131
  loss = None
132
  if targets is not None:
133
+ # targets with value -100 are automatically ignored in cross_entropy
134
  loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-100)
135
  return logits, loss
code/train.py CHANGED
@@ -1,387 +1,367 @@
1
  import os
2
- os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
 
 
 
 
 
 
 
 
 
 
3
 
4
- import json, math, time, random, sys, contextlib
5
- import numpy as np
6
  import torch
7
-
8
- torch.set_float32_matmul_precision('high')
9
- torch.backends.cuda.matmul.allow_tf32 = True
10
- torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = True
11
-
12
  import torch.distributed as dist
13
  from torch.nn.parallel import DistributedDataParallel as DDP
14
- from huggingface_hub import HfApi, hf_hub_download, create_repo, login
15
- from datetime import timedelta
16
-
17
- def setup_ddp():
18
- dist.init_process_group(backend="nccl", timeout=timedelta(minutes=30))
19
- rank = dist.get_rank()
20
- local_rank = int(os.environ["LOCAL_RANK"])
21
- world_size = dist.get_world_size()
22
- torch.cuda.set_device(local_rank)
23
- return rank, local_rank, world_size
24
 
25
- rank, local_rank, world_size = setup_ddp()
26
- DEVICE = f"cuda:{local_rank}"
27
- IS_MAIN = rank == 0
 
 
28
 
29
- def log(msg):
30
- if IS_MAIN:
31
- print(msg, flush=True)
32
 
33
  def get_hf_token():
34
  token = os.environ.get("HF_TOKEN")
35
  if token:
 
36
  return token
37
- try:
38
- from kaggle_secrets import UserSecretsClient
39
- token = UserSecretsClient().get_secret("HF_TOKEN")
40
- if token:
41
- return token
42
- except Exception:
43
- pass
44
- raise ValueError("HF_TOKEN environment variable or Kaggle Secret is not set!")
45
-
46
- HF_TOKEN = get_hf_token()
47
- login(token=HF_TOKEN)
48
- api = HfApi(token=HF_TOKEN)
49
-
50
- from transformers import AutoTokenizer
51
-
52
- DATA_REPO = "ViuAI/viuai-500m-data"
53
- MODEL_REPO = "ViuAI/ViuRec"
54
- TOKENIZER_SUBFOLDER = "tokenizer"
55
- CODE_SUBFOLDER = "code"
56
- CKPT_SUBFOLDER = "checkpoints"
57
- CKPT_REPO = MODEL_REPO
58
 
59
- def load_tokenizer_robust():
60
- paths_to_try = [
61
- "/kaggle/working/ViuRec/tokenizer",
62
- "/kaggle/input/viuai-500m-tokenizer",
63
- "./viuai-500m-tokenizer"
64
- ]
65
- for p in paths_to_try:
66
- if os.path.exists(os.path.join(p, "vocab.json")):
67
- try:
68
- tok = AutoTokenizer.from_pretrained(p)
69
- if IS_MAIN:
70
- print(f"✅ Tokenizer loaded from {p}")
71
- return tok
72
- except Exception:
73
- pass
 
 
 
74
  try:
75
- tok = AutoTokenizer.from_pretrained(MODEL_REPO, subfolder=TOKENIZER_SUBFOLDER, token=HF_TOKEN, use_fast=False)
76
- if IS_MAIN:
77
- print(f"✅ Tokenizer loaded from HF ({MODEL_REPO}/{TOKENIZER_SUBFOLDER})")
78
- return tok
79
  except Exception as e:
80
- print(f"⚠️ Tokenizer bypassed.")
81
- return None
82
-
83
- tokenizer = load_tokenizer_robust()
84
- VOCAB_SIZE = len(tokenizer) if tokenizer is not None else 64000
85
- if IS_MAIN: print(f"ℹ️ VOCAB_SIZE set to {VOCAB_SIZE}")
86
-
87
- if IS_MAIN:
88
- create_repo(CKPT_REPO, repo_type="model", token=HF_TOKEN, exist_ok=True, private=True)
89
- dist.barrier()
90
-
91
- if os.path.exists("/kaggle/working"):
92
- WORK = "/kaggle/working"
93
- elif os.path.exists("/root/working"):
94
- WORK = "/root/working"
95
- else:
96
- WORK = "./working"
97
- os.makedirs(WORK, exist_ok=True)
98
-
99
- sys.path.insert(0, WORK)
100
- sys.path.insert(0, os.path.join(WORK, CODE_SUBFOLDER))
101
- hf_hub_download(MODEL_REPO, f"{CODE_SUBFOLDER}/model.py", local_dir=WORK, token=HF_TOKEN)
102
- from model import ViuResonance100M
103
 
104
- def with_retry(fn, *args, retries=4, delay=5, **kwargs):
105
- for attempt in range(1, retries + 1):
 
106
  try:
107
- return fn(*args, **kwargs)
108
  except Exception as e:
109
- if "404" in str(e) or "EntryNotFoundError" in type(e).__name__:
110
- raise
111
- if attempt == retries:
112
- raise
113
- log(f" [retry {attempt}/{retries}] {type(e).__name__}: {e}")
114
- time.sleep(delay)
115
- delay *= 2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
116
 
117
  class ShardPool:
118
- def __init__(self, shard_names, repo_id, token, local_dir, pool_size=3, seed=0):
119
- self.all_shards = list(shard_names)
120
- self.rng = random.Random(seed)
121
- self.rng.shuffle(self.all_shards)
122
- self.cursor = 0
123
- self.repo_id, self.token, self.local_dir, self.pool_size = repo_id, token, local_dir, pool_size
124
- os.makedirs(local_dir, exist_ok=True)
125
  self.pool = []
126
- self.fill()
127
-
128
- def _next_name(self):
129
- if self.cursor >= len(self.all_shards):
130
- self.rng.shuffle(self.all_shards)
131
- self.cursor = 0
132
- name = self.all_shards[self.cursor]
133
- self.cursor += 1
134
- return name
135
-
136
- def fill(self):
137
- while len(self.pool) < self.pool_size:
138
- name = self._next_name()
139
- path = with_retry(hf_hub_download, self.repo_id, name, repo_type="dataset",
140
- local_dir=self.local_dir, token=self.token)
141
- mmap_arr = np.memmap(path, dtype=np.uint16, mode="r")
142
- if len(mmap_arr) < 2049: # Context length (2048) + 1
143
- if IS_MAIN:
144
- print(f"⚠️ Skipping shard {name} as it is too short ({len(mmap_arr)} tokens).")
 
 
 
 
 
 
 
 
 
 
145
  continue
146
- self.pool.append((path, mmap_arr))
147
-
148
- def rotate_one(self):
149
- old_path, old_mmap = self.pool.pop(0)
150
- del old_mmap
151
- try: os.remove(old_path)
152
- except OSError: pass
153
- self.fill()
154
-
155
- def sample_batch(self, batch_size, ctx_len):
156
  xs, ys = [], []
157
  while len(xs) < batch_size:
158
- _, tokens = self.rng.choice(self.pool)
 
159
  max_start = len(tokens) - ctx_len - 1
160
  if max_start <= 0:
161
  continue
162
- i = self.rng.randint(0, max_start)
163
  xs.append(np.array(tokens[i:i+ctx_len], dtype=np.int64))
164
  ys.append(np.array(tokens[i+1:i+1+ctx_len], dtype=np.int64))
165
  return np.stack(xs), np.stack(ys)
166
-
167
- def load_val_tokens(val_names, repo_id, token, local_dir):
168
- os.makedirs(local_dir, exist_ok=True)
169
- arrays = []
170
- for name in val_names:
171
- path = with_retry(hf_hub_download, repo_id, name, repo_type="dataset", local_dir=local_dir, token=token)
172
- arrays.append(np.memmap(path, dtype=np.uint16, mode="r"))
173
- return np.concatenate(arrays)
174
-
175
- def get_batch(source, batch_size, ctx_len):
176
- if isinstance(source, ShardPool):
177
- x_np, y_np = source.sample_batch(batch_size, ctx_len)
178
- else:
179
- i = np.random.randint(0, len(source) - ctx_len - 1, size=batch_size)
180
- x_np = np.stack([np.array(source[j:j+ctx_len], dtype=np.int64) for j in i])
181
- y_np = np.stack([np.array(source[j+1:j+1+ctx_len], dtype=np.int64) for j in i])
182
- x = torch.from_numpy(x_np).to(DEVICE, non_blocking=True)
183
- y = torch.from_numpy(y_np).to(DEVICE, non_blocking=True)
184
- return x, y
185
-
186
- def extract_clean_state_dict(model):
187
- raw_model = model.module if hasattr(model, "module") else model
188
- state_dict = raw_model.state_dict()
189
- return {k.replace("_orig_mod.", ""): v for k, v in state_dict.items()}
190
-
191
- def save_checkpoint(model, optimizer, step, val_loss, filename):
192
- if not IS_MAIN:
193
- return
194
- path = f"{WORK}/{filename}"
195
- state_dict = extract_clean_state_dict(model)
196
- data = {"model": state_dict, "step": step, "val_loss": val_loss}
197
-
198
- if filename == "ckpt_latest.pt" and optimizer is not None:
199
- data["optimizer"] = optimizer.state_dict()
200
 
201
- torch.save(data, path)
202
- with_retry(api.upload_file, path_or_fileobj=path, path_in_repo=f"{CKPT_SUBFOLDER}/{filename}",
203
- repo_id=CKPT_REPO, repo_type="model", token=HF_TOKEN)
204
- log(f" [{filename} saved @ step {step}, val_loss={val_loss:.4f}]")
205
-
206
- def load_checkpoint(model, optimizer):
207
- try:
208
  try:
209
- path = with_retry(hf_hub_download, CKPT_REPO, f"{CKPT_SUBFOLDER}/ckpt_latest.pt", repo_type="model",
210
- local_dir=WORK, token=HF_TOKEN, retries=1)
211
  except Exception as e:
212
- if "404" in str(e) or "EntryNotFoundError" in type(e).__name__:
213
- log("No checkpoint found on HF — starting from scratch.")
214
- return 0
215
- raise
216
-
217
- ckpt = torch.load(path, map_location=DEVICE, weights_only=False)
218
- target = model.module._orig_mod if hasattr(model.module, '_orig_mod') else model.module
219
- target.load_state_dict(ckpt["model"])
220
- if optimizer is not None and "optimizer" in ckpt:
221
- optimizer.load_state_dict(ckpt["optimizer"])
222
- log(f"Resumed from step {ckpt['step']} (val_loss={ckpt['val_loss']:.4f})")
223
- return ckpt["step"]
224
- except Exception as e:
225
- log(f"Failed to load checkpoint — starting fresh. ({type(e).__name__}: {e})")
226
- return 0
227
-
228
- def get_lr(step, warmup, total, max_lr, min_lr):
229
- if step < warmup:
230
- return max_lr * (step + 1) / warmup
231
- if step > total:
232
- return min_lr
233
- ratio = (step - warmup) / (total - warmup)
234
- return min_lr + 0.5 * (1 + math.cos(math.pi * ratio)) * (max_lr - min_lr)
235
-
236
- # === GPU CONFIG ===
237
- mem_gb = torch.cuda.get_device_properties(DEVICE).total_memory / (1024**3)
238
- if mem_gb > 100: # H200
239
- MICRO_BATCH = 32
240
- GRAD_ACCUM = 4
241
- elif mem_gb > 70: # A100
242
- MICRO_BATCH = 32
243
- GRAD_ACCUM = 4
244
- else: # T4 or RTX (15GB/24GB)
245
- MICRO_BATCH = 1
246
- GRAD_ACCUM = 32
247
-
248
- if IS_MAIN:
249
- print(f"🖥️ Detected GPU with {mem_gb:.1f}GB VRAM -> Auto-Configured: MICRO_BATCH={MICRO_BATCH}, GRAD_ACCUM={GRAD_ACCUM}")
250
- # ==================
251
-
252
- CONTEXT_LEN = 2048
253
- MAX_LR = 3e-4
254
- MIN_LR = 3e-5
255
- WARMUP_STEPS = 1000
256
- TOTAL_STEPS = 40_000
257
- WEIGHT_DECAY = 0.1
258
- GRAD_CLIP = 1.0
259
- MAX_STEPS = int(os.environ.get("MAX_STEPS", TOTAL_STEPS))
260
- EVAL_EVERY = 500
261
- SAVE_EVERY = 2000
262
- ROTATE_EVERY = 500
263
-
264
- torch.backends.cudnn.benchmark = False
265
-
266
- # Model 100M init
267
- model = ViuResonance100M(vocab_size=VOCAB_SIZE, d_model=1024, n_layers=12, n_heads=16).to(DEVICE)
268
-
269
- if IS_MAIN:
270
- log(f"Model params: {sum(p.numel() for p in model.parameters())/1e6:.1f}M | world_size={world_size}")
271
- log("Bypassing torch.compile (Disabled for Kaggle T4 compatibility)...")
272
 
273
- # torch.compile removed to prevent Kaggle T4 IndentationError/freeze
274
- model = DDP(model, device_ids=[local_rank])
 
 
 
 
275
 
276
- optimizer = torch.optim.AdamW(model.parameters(), lr=MAX_LR, weight_decay=WEIGHT_DECAY, betas=(0.9, 0.95), fused=True)
 
 
 
 
 
277
 
278
- start_step = load_checkpoint(model, optimizer)
279
- start_step_t = torch.tensor([start_step], device=DEVICE)
280
- dist.broadcast(start_step_t, src=0)
281
- start_step = start_step_t.item()
282
 
283
- files = api.list_repo_files(DATA_REPO, repo_type="dataset")
284
- train_shards = sorted(f for f in files if f.startswith("shards/") and f.endswith(".bin") and "_val" not in f)
285
- val_shards = sorted(f for f in files if f.startswith("shards/") and f.endswith("_val.bin"))
286
 
287
- shard_pool = ShardPool(train_shards, DATA_REPO, HF_TOKEN, f"{WORK}/train_shards_r{rank}", seed=rank + start_step)
288
- val_tokens = load_val_tokens(val_shards, DATA_REPO, HF_TOKEN, f"{WORK}/val_shards") if IS_MAIN else None
289
- dist.barrier() # Sync ranks after downloading validation data
290
 
291
- @torch.no_grad()
292
- def evaluate():
293
- if not IS_MAIN:
294
- return None
295
- model.eval()
296
- losses = []
297
- for _ in range(20):
298
- x, y = get_batch(val_tokens, 16, CONTEXT_LEN)
299
- with torch.autocast(device_type="cuda", dtype=torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16):
300
- unwrapped = model.module._orig_mod if hasattr(model.module, '_orig_mod') else model.module
301
- _, loss = unwrapped(x, y)
302
- losses.append(loss.item())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
303
  model.train()
304
- return sum(losses) / len(losses)
305
-
306
- model.train()
307
- val_loss = None
308
- best_val_loss = None
309
- ema_tok_s = None
310
- global_t0 = time.time()
311
- t0 = time.time()
312
- end_step = min(start_step + MAX_STEPS, TOTAL_STEPS)
313
-
314
- for step in range(start_step, end_step):
315
- lr = get_lr(step, WARMUP_STEPS, TOTAL_STEPS, MAX_LR, MIN_LR)
316
- for g in optimizer.param_groups:
317
- g["lr"] = lr
318
-
319
- optimizer.zero_grad(set_to_none=True)
320
- accum_loss = 0.0
321
- for micro_i in range(GRAD_ACCUM):
322
- x, y = get_batch(shard_pool, MICRO_BATCH, CONTEXT_LEN)
323
- with torch.autocast(device_type="cuda", dtype=torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16):
324
- _, loss = model(x, y)
325
- loss = loss / GRAD_ACCUM
326
- loss.backward()
327
- accum_loss += loss.item()
328
- del x, y, loss
329
-
330
- torch.nn.utils.clip_grad_norm_(model.parameters(), GRAD_CLIP)
331
 
332
- if not math.isfinite(accum_loss):
333
- if IS_MAIN:
334
- log(f"⚠️ NaN/Inf loss at step {step}! Skipping optimizer step.")
335
  optimizer.zero_grad(set_to_none=True)
336
- continue
337
-
338
- optimizer.step()
339
-
340
- if IS_MAIN and step % 10 == 0:
341
- if step == start_step:
342
- t0 = time.time()
343
- mem_alloc = torch.cuda.memory_allocated(DEVICE) / 1e9
344
- mem_reserved = torch.cuda.memory_reserved(DEVICE) / 1e9
345
- log(f"step {step:5d}/{end_step} | loss {accum_loss:.4f} | lr {lr:.2e} | Calibrating speed... | "
346
- f"mem alloc={mem_alloc:.2f}GB reserved={mem_reserved:.2f}GB")
347
- else:
348
- dt = time.time() - t0
349
- tok_s = (MICRO_BATCH * GRAD_ACCUM * CONTEXT_LEN * world_size * 10) / dt if dt > 0 else 0
350
- ema_tok_s = tok_s if ema_tok_s is None else 0.7 * ema_tok_s + 0.3 * tok_s
351
- t0 = time.time()
352
- mem_alloc = torch.cuda.memory_allocated(DEVICE) / 1e9
353
- mem_reserved = torch.cuda.memory_reserved(DEVICE) / 1e9
354
- steps_done = step - start_step
355
- if steps_done > 0:
356
- elapsed = time.time() - global_t0
357
- eta_s = elapsed / steps_done * (end_step - step)
358
- eta_str = f"{eta_s/60:.0f}min" if eta_s < 3600 else f"{eta_s/3600:.1f}hr"
359
- else:
360
- eta_str = "..."
361
- log(f"step {step:5d}/{end_step} | loss {accum_loss:.4f} | lr {lr:.2e} | {ema_tok_s:.0f} tok/s | "
362
- f"ETA {eta_str} | mem alloc={mem_alloc:.2f}GB reserved={mem_reserved:.2f}GB")
363
-
364
- if step > start_step and step % ROTATE_EVERY == 0:
365
- shard_pool.rotate_one()
366
-
367
- if step > start_step and step % EVAL_EVERY == 0:
368
- val_loss = evaluate()
369
- if IS_MAIN and val_loss is not None:
370
- log(f"step {step} | val_loss {val_loss:.4f}")
371
- if best_val_loss is None or val_loss < best_val_loss:
372
- best_val_loss = val_loss
373
-
374
- if step > start_step and step % SAVE_EVERY == 0:
375
- dist.barrier()
376
- if IS_MAIN:
377
- save_checkpoint(model.module._orig_mod if hasattr(model.module, '_orig_mod') else model.module,
378
- optimizer, step, val_loss if val_loss is not None else -1, "ckpt_latest.pt")
379
-
380
- dist.barrier()
381
- final_val = evaluate()
382
- if IS_MAIN:
383
- save_checkpoint(model.module._orig_mod if hasattr(model.module, '_orig_mod') else model.module,
384
- optimizer, end_step, final_val if final_val is not None else -1, "ckpt_latest.pt")
385
- log(f"\n100M Training complete! Final val_loss: {final_val if final_val is not None else -1:.4f}")
386
-
387
- dist.destroy_process_group()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import os
2
+ # Setup env vars for robust Kaggle training
3
+ os.environ["NCCL_P2P_DISABLE"] = "1"
4
+ os.environ["NCCL_IB_DISABLE"] = "1"
5
+ os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
6
+ os.environ["HF_HUB_DISABLE_PROGRESS_BARS"] = "1"
7
+
8
+ import math
9
+ import time
10
+ import argparse
11
+ import logging
12
+ from pathlib import Path
13
 
 
 
14
  import torch
 
 
 
 
 
15
  import torch.distributed as dist
16
  from torch.nn.parallel import DistributedDataParallel as DDP
17
+ from torch.cuda.amp import GradScaler
18
+ import numpy as np
19
+ from transformers import AutoTokenizer
20
+ from huggingface_hub import hf_hub_download, HfApi
21
+ from huggingface_hub.utils import EntryNotFoundError, HfHubHTTPError
 
 
 
 
 
22
 
23
+ try:
24
+ from kaggle_secrets import UserSecretsClient
25
+ HAS_KAGGLE_SECRETS = True
26
+ except ImportError:
27
+ HAS_KAGGLE_SECRETS = False
28
 
29
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
30
+ logger = logging.getLogger(__name__)
 
31
 
32
  def get_hf_token():
33
  token = os.environ.get("HF_TOKEN")
34
  if token:
35
+ logger.info("Loaded HF_TOKEN from environment.")
36
  return token
37
+ if HAS_KAGGLE_SECRETS:
38
+ try:
39
+ token = UserSecretsClient().get_secret("HF_TOKEN")
40
+ if token:
41
+ os.environ["HF_TOKEN"] = token
42
+ logger.info("Loaded HF_TOKEN from kaggle_secrets.")
43
+ return token
44
+ except Exception as e:
45
+ logger.warning(f"Failed to get HF_TOKEN from kaggle_secrets: {e}")
46
+ raise ValueError("HF_TOKEN not found in environment or kaggle_secrets.")
 
 
 
 
 
 
 
 
 
 
 
47
 
48
+ def setup_ddp():
49
+ if "RANK" in os.environ and "WORLD_SIZE" in os.environ:
50
+ dist.init_process_group("nccl")
51
+ rank = int(os.environ["RANK"])
52
+ local_rank = int(os.environ["LOCAL_RANK"])
53
+ world_size = int(os.environ["WORLD_SIZE"])
54
+ is_ddp = True
55
+ else:
56
+ rank = 0
57
+ local_rank = 0
58
+ world_size = 1
59
+ is_ddp = False
60
+ device = torch.device(f"cuda:{local_rank}")
61
+ torch.cuda.set_device(device)
62
+ return rank, local_rank, world_size, device, is_ddp
63
+
64
+ def load_tokenizer_robust(token: str):
65
+ logger.info("Loading tokenizer...")
66
  try:
67
+ return AutoTokenizer.from_pretrained("ViuAI/ViuRec", token=token, use_fast=True)
 
 
 
68
  except Exception as e:
69
+ logger.warning(f"Failed to load fast tokenizer: {e}. Trying use_fast=False.")
70
+ try:
71
+ return AutoTokenizer.from_pretrained("ViuAI/ViuRec", token=token, use_fast=False)
72
+ except Exception as e2:
73
+ logger.error(f"Failed to load tokenizer: {e2}")
74
+ raise
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
75
 
76
+ def load_model_code(token: str):
77
+ if not Path("model.py").exists():
78
+ logger.info("model.py not found locally. Downloading from HuggingFace...")
79
  try:
80
+ hf_hub_download(repo_id="ViuAI/ViuRec", filename="code/model.py", token=token, local_dir=".")
81
  except Exception as e:
82
+ logger.error(f"Failed to download model.py: {e}")
83
+ raise
84
+
85
+ def load_checkpoint(model, optimizer, scaler, api, token, repo_id, filename="checkpoints/latest_chk.pt", strict=False):
86
+ local_path = "latest_chk.pt"
87
+ try:
88
+ logger.info(f"Checking for existing checkpoint on HF: {filename}")
89
+ hf_hub_download(repo_id=repo_id, filename=filename, local_dir=".", token=token)
90
+ # Security Note: weights_only=False is used here to allow loading the optimizer state safely inside our trusted environment.
91
+ ckpt = torch.load(local_path, map_location="cpu", weights_only=False)
92
+ missing, unexpected = model.load_state_dict(ckpt["model"], strict=strict)
93
+ if missing or unexpected:
94
+ logger.warning(f"Checkpoint loaded with Missing keys: {missing} | Unexpected keys: {unexpected}")
95
+ else:
96
+ logger.info("Model weights loaded successfully.")
97
+
98
+ optimizer.load_state_dict(ckpt["optimizer"])
99
+ if "scaler" in ckpt and scaler is not None:
100
+ scaler.load_state_dict(ckpt["scaler"])
101
+
102
+ step = ckpt.get("step", 0)
103
+ logger.info(f"Resuming from step {step}")
104
+ Path(local_path).unlink(missing_ok=True)
105
+ return step
106
+ except (EntryNotFoundError, HfHubHTTPError):
107
+ logger.info("No checkpoint found on HF. Starting from scratch.")
108
+ return 0
109
+ except Exception as e:
110
+ logger.error(f"Error loading checkpoint: {e}")
111
+ return 0
112
 
113
  class ShardPool:
114
+ def __init__(self, api, repo_id, work_dir, token, vocab_size, max_pool=3):
115
+ self.api = api
116
+ self.repo_id = repo_id
117
+ self.work_dir = Path(work_dir)
118
+ self.work_dir.mkdir(parents=True, exist_ok=True)
119
+ self.token = token
120
+ self.max_pool = max_pool
121
  self.pool = []
122
+ self.available_files = []
123
+ self.vocab_size = vocab_size
124
+ self._refresh_file_list()
125
+
126
+ def _refresh_file_list(self):
127
+ files = self.api.list_repo_files(repo_id=self.repo_id, repo_type="dataset", token=self.token)
128
+ self.available_files = [f for f in files if f.startswith("shards/hi_indiccorp_") and f.endswith(".bin")]
129
+ np.random.shuffle(self.available_files)
130
+
131
+ def fill(self, ctx_len):
132
+ max_attempts = self.max_pool * 5
133
+ attempts = 0
134
+ while len(self.pool) < self.max_pool:
135
+ if attempts >= max_attempts:
136
+ raise RuntimeError("Failed to fill ShardPool: too many invalid shards.")
137
+ if not self.available_files:
138
+ self._refresh_file_list()
139
+ f = self.available_files.pop()
140
+ local_p = hf_hub_download(repo_id=self.repo_id, repo_type="dataset", filename=f, local_dir=self.work_dir, token=self.token)
141
+
142
+ # Safe dtype selection
143
+ dtype = np.uint16 if self.vocab_size <= 65535 else np.uint32
144
+ mmap_arr = np.memmap(local_p, dtype=dtype, mode="r")
145
+
146
+ if len(mmap_arr) < ctx_len + 1:
147
+ logger.warning(f"Shard {f} is too small ({len(mmap_arr)} tokens). Discarding.")
148
+ del mmap_arr
149
+ Path(local_p).unlink(missing_ok=True)
150
+ attempts += 1
151
  continue
152
+
153
+ self.pool.append((local_p, mmap_arr))
154
+ logger.info(f"Loaded shard {f}")
155
+
156
+ def sample_batch(self, batch_size, ctx_len, rng):
 
 
 
 
 
157
  xs, ys = [], []
158
  while len(xs) < batch_size:
159
+ idx = rng.integers(0, len(self.pool))
160
+ _, tokens = self.pool[idx]
161
  max_start = len(tokens) - ctx_len - 1
162
  if max_start <= 0:
163
  continue
164
+ i = rng.integers(0, max_start)
165
  xs.append(np.array(tokens[i:i+ctx_len], dtype=np.int64))
166
  ys.append(np.array(tokens[i+1:i+1+ctx_len], dtype=np.int64))
167
  return np.stack(xs), np.stack(ys)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
168
 
169
+ def rotate_one(self):
170
+ if not self.pool: return
171
+ old_path, old_mmap = self.pool.pop(0)
172
+ del old_mmap
 
 
 
173
  try:
174
+ Path(old_path).unlink(missing_ok=True)
175
+ logger.info(f"Rotated out shard {Path(old_path).name}")
176
  except Exception as e:
177
+ logger.warning(f"Failed to delete {old_path}: {e}")
178
+
179
+ def main():
180
+ parser = argparse.ArgumentParser()
181
+ parser.add_argument("--max-steps", type=int, default=100)
182
+ parser.add_argument("--micro-batch", type=int, default=2)
183
+ parser.add_argument("--grad-accum", type=int, default=16)
184
+ parser.add_argument("--eval-every", type=int, default=500)
185
+ parser.add_argument("--save-every", type=int, default=500)
186
+ parser.add_argument("--rotate-every", type=int, default=500)
187
+ parser.add_argument("--lr", type=float, default=3e-4)
188
+ parser.add_argument("--weight-decay", type=float, default=0.1)
189
+ parser.add_argument("--ctx-len", type=int, default=2048)
190
+ args = parser.parse_args()
191
+
192
+ token = get_hf_token()
193
+ rank, local_rank, world_size, device, is_ddp = setup_ddp()
194
+ is_main = (rank == 0)
195
+
196
+ # Random seed
197
+ seed = 42 + rank
198
+ torch.manual_seed(seed)
199
+ np.random.seed(seed)
200
+ rng = np.random.default_rng(seed)
201
+
202
+ # Load resources
203
+ tokenizer = load_tokenizer_robust(token)
204
+ vocab_size = len(tokenizer)
205
+ load_model_code(token)
206
+
207
+ from model import ViuResonance100M, ModelConfig
208
+ config = ModelConfig(vocab_size=vocab_size)
209
+ model = ViuResonance100M(config).to(device)
210
+
211
+ if is_ddp:
212
+ model = DDP(model, device_ids=[local_rank], output_device=local_rank, broadcast_buffers=False)
213
+ unwrapped_model = model.module
214
+ else:
215
+ unwrapped_model = model
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
216
 
217
+ # Optional torch.compile fallback
218
+ try:
219
+ model = torch.compile(model)
220
+ if is_main: logger.info("Compiled model successfully.")
221
+ except Exception as e:
222
+ if is_main: logger.warning(f"Failed to compile model: {e}. Falling back to eager mode.")
223
 
224
+ # Optimizer
225
+ try:
226
+ optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=args.weight_decay, betas=(0.9, 0.95), fused=True)
227
+ except:
228
+ if is_main: logger.warning("fused=True failed for AdamW. Falling back to non-fused.")
229
+ optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=args.weight_decay, betas=(0.9, 0.95))
230
 
231
+ # GradScaler for safe fp16/bf16
232
+ scaler = GradScaler(enabled=not torch.cuda.is_bf16_supported())
 
 
233
 
234
+ api = HfApi()
235
+ start_step = load_checkpoint(unwrapped_model, optimizer, scaler, api, token, "ViuAI/ViuRec", strict=False)
 
236
 
237
+ work_dir = Path("/kaggle/working" if Path("/kaggle/working").exists() else ".")
238
+ shard_pool = ShardPool(api, "ViuAI/ViuCorp", work_dir / "shards", token, vocab_size)
239
+ shard_pool.fill(args.ctx_len)
240
 
241
+ # Val Data Safely (cap at 20M tokens)
242
+ val_tokens = None
243
+ if is_main:
244
+ logger.info("Loading validation data...")
245
+ try:
246
+ val_files = [f for f in api.list_repo_files(repo_id="ViuAI/ViuCorp", repo_type="dataset", token=token) if f.startswith("val_shards/")]
247
+ if val_files:
248
+ val_p = hf_hub_download(repo_id="ViuAI/ViuCorp", repo_type="dataset", filename=val_files[0], local_dir=work_dir, token=token)
249
+ dtype = np.uint16 if vocab_size <= 65535 else np.uint32
250
+ val_arr = np.memmap(val_p, dtype=dtype, mode="r")
251
+ max_toks = 20_000_000
252
+ val_tokens = np.array(val_arr[:max_toks]) # Copy to RAM, safely capped
253
+ del val_arr
254
+ Path(val_p).unlink(missing_ok=True)
255
+ except Exception as e:
256
+ logger.warning(f"Failed to load val data: {e}")
257
+
258
+ @torch.no_grad()
259
+ def evaluate():
260
+ if val_tokens is None or len(val_tokens) < args.ctx_len + 1: return 0.0
261
+ unwrapped_model.eval()
262
+ val_loss = 0.0
263
+ val_iters = 20
264
+ for _ in range(val_iters):
265
+ i = rng.integers(0, len(val_tokens) - args.ctx_len - 1)
266
+ x = torch.from_numpy(val_tokens[i:i+args.ctx_len].astype(np.int64)).unsqueeze(0).to(device, non_blocking=True)
267
+ y = torch.from_numpy(val_tokens[i+1:i+1+args.ctx_len].astype(np.int64)).unsqueeze(0).to(device, non_blocking=True)
268
+ with torch.autocast(device_type="cuda", dtype=torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16):
269
+ # Pass use_checkpoint=False for evaluation
270
+ _, loss = unwrapped_model(x, y, use_checkpoint=False)
271
+ val_loss += loss.item()
272
+ unwrapped_model.train()
273
+ return val_loss / val_iters
274
+
275
+ # Training Loop
276
  model.train()
277
+ if is_main:
278
+ logger.info(f"Starting training for {args.max_steps} steps...")
279
+
280
+ for step in range(start_step, args.max_steps):
281
+ # LR Schedule
282
+ warmup = 1000
283
+ progress = min(1.0, step / max(1, warmup))
284
+ lr = args.lr * progress if step < warmup else args.lr * (0.5 * (1.0 + math.cos(math.pi * (step - warmup) / max(1, args.max_steps - warmup))))
285
+ for param_group in optimizer.param_groups:
286
+ param_group['lr'] = lr
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
287
 
 
 
 
288
  optimizer.zero_grad(set_to_none=True)
289
+ accum_loss = 0.0
290
+
291
+ t0 = time.time()
292
+ for micro_i in range(args.grad_accum):
293
+ x_np, y_np = shard_pool.sample_batch(args.micro_batch, args.ctx_len, rng)
294
+ x = torch.from_numpy(x_np).to(device, non_blocking=True)
295
+ y = torch.from_numpy(y_np).to(device, non_blocking=True)
296
+
297
+ with torch.autocast(device_type="cuda", dtype=torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16):
298
+ _, loss = model(x, y)
299
+ loss = loss / args.grad_accum
300
+
301
+ scaler.scale(loss).backward()
302
+ accum_loss += loss.item()
303
+ del x, y, loss
304
+
305
+ # Safe Gradient Clipping with Scaler
306
+ scaler.unscale_(optimizer)
307
+ torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
308
+
309
+ # Check for NaNs
310
+ has_nan = not math.isfinite(accum_loss)
311
+ for p in model.parameters():
312
+ if p.grad is not None and not torch.isfinite(p.grad).all():
313
+ has_nan = True
314
+ break
315
+
316
+ if has_nan:
317
+ if is_main: logger.warning(f"NaN/Inf detected at step {step}! Skipping optimizer step.")
318
+ optimizer.zero_grad(set_to_none=True)
319
+ continue
320
+
321
+ scaler.step(optimizer)
322
+ scaler.update()
323
+
324
+ dt = time.time() - t0
325
+
326
+ if is_main and step % 10 == 0:
327
+ tok_s = (args.micro_batch * args.grad_accum * args.ctx_len * world_size) / dt if dt > 0 else 0
328
+ mem_alloc = torch.cuda.memory_allocated(device) / 1e9
329
+ mem_res = torch.cuda.memory_reserved(device) / 1e9
330
+ logger.info(f"step {step:5d}/{args.max_steps} | loss {accum_loss:.4f} | lr {lr:.2e} | {tok_s:.0f} tok/s | mem alloc={mem_alloc:.2f}GB res={mem_res:.2f}GB")
331
+
332
+ if step > start_step and step % args.rotate_every == 0:
333
+ shard_pool.rotate_one()
334
+ shard_pool.fill(args.ctx_len)
335
+
336
+ if step > start_step and step % args.eval_every == 0 and is_main:
337
+ val_loss = evaluate()
338
+ logger.info(f"--- Eval step {step}: val_loss {val_loss:.4f} ---")
339
+
340
+ if step > start_step and step % args.save_every == 0:
341
+ if is_ddp: dist.barrier()
342
+ if is_main:
343
+ logger.info(f"Saving checkpoint at step {step}...")
344
+ chk = {
345
+ "model": unwrapped_model.state_dict(),
346
+ "optimizer": optimizer.state_dict(),
347
+ "scaler": scaler.state_dict(),
348
+ "step": step,
349
+ }
350
+ torch.save(chk, "latest_chk.pt")
351
+ try:
352
+ api.upload_file(
353
+ path_or_fileobj="latest_chk.pt",
354
+ path_in_repo="checkpoints/latest_chk.pt",
355
+ repo_id="ViuAI/ViuRec",
356
+ token=token
357
+ )
358
+ logger.info("Upload successful.")
359
+ except Exception as e:
360
+ logger.error(f"Upload failed: {e}")
361
+ if is_ddp: dist.barrier()
362
+
363
+ if is_ddp:
364
+ dist.destroy_process_group()
365
+
366
+ if __name__ == "__main__":
367
+ main()