Mouserat commited on
Commit
c1f76eb
·
verified ·
1 Parent(s): c2b5bfe

Upload notebooks/rat77m_train_ready.ipynb with huggingface_hub

Browse files
Files changed (1) hide show
  1. notebooks/rat77m_train_ready.ipynb +449 -637
notebooks/rat77m_train_ready.ipynb CHANGED
@@ -1,94 +1,117 @@
1
  {
2
  "nbformat": 4,
3
- "nbformat_minor": 0,
4
  "metadata": {
5
  "kernelspec": {
6
- "name": "python3",
7
- "display_name": "Python 3"
 
8
  },
9
  "language_info": {
10
  "name": "python"
11
  },
12
- "accelerator": "GPU"
 
 
13
  },
14
  "cells": [
15
  {
16
  "cell_type": "markdown",
17
  "metadata": {},
18
  "source": [
19
- "# rat-77m 事前学習 (G4) — 修正版\n",
20
- "\n",
21
- "## 前回失敗 + 今回レビューの反映\n",
22
- "| 項目 | 内容 |\n",
23
- "|---|---|\n",
24
- "| emb | **行 L2=0.5**(行 L2≈11.3)。仕様書 v6 手順に合わせる |\n",
25
- "| データ ID | .bin は **mT5 旧 ID** → 学習前に **64k 新 ID へ再マップ**(必須) |\n",
26
- "| LR | Adafactor **1e-4** |\n",
27
- "| 診断 | 学習前に loss≈11 / logits std / grad_norm |\n",
28
- "| 停止 | grad_norm>50 が **連続2回** |\n",
29
- "| Discord | `DISCORD_WEBHOOK_URL` で 500 step ごと |\n",
30
- "| Bucket | 5000 ごと latest 置き換え / 完了時 final-20 |\n",
31
- "\n",
32
- "環境変数: `HF_TOKEN` / 任意 `DISCORD_WEBHOOK_URL`\n",
33
- ""
 
 
 
 
 
 
 
 
 
 
 
 
 
34
  ]
35
  },
36
  {
37
  "cell_type": "code",
38
  "metadata": {},
 
 
39
  "source": [
40
  "#@title 1. 環境セットアップ\n",
41
- "\n",
42
- "import os, subprocess, sys, math, json, time, shutil\n",
43
  "from pathlib import Path\n",
44
  "\n",
45
- "def run(cmd):\n",
46
- " print(\"+\", cmd)\n",
47
- " subprocess.check_call(cmd, shell=True)\n",
48
  "\n",
49
- "run(\"pip install -q transformers datasets accelerate sentencepiece huggingface_hub safetensors\")\n",
50
  "\n",
51
- "import torch\n",
52
  "import numpy as np\n",
53
- "print(\"torch:\", torch.__version__, \"cuda:\", torch.cuda.is_available())\n",
54
- "if torch.cuda.is_available():\n",
55
- " print(torch.cuda.get_device_name(0), \"VRAM GB:\", round(torch.cuda.get_device_properties(0).total_memory/1024**3, 1))\n",
56
  "\n",
57
- "HF_TOKEN = os.environ.get(\"HF_TOKEN\") or os.environ.get(\"HUGGING_FACE_HUB_TOKEN\")\n",
58
- "if not HF_TOKEN:\n",
59
- " try:\n",
60
- " from google.colab import userdata\n",
61
- " HF_TOKEN = userdata.get(\"HF_TOKEN\")\n",
62
- " except Exception:\n",
63
- " pass\n",
64
- "assert HF_TOKEN, \"HF_TOKEN を設定してください\"\n",
65
- "os.environ[\"HF_TOKEN\"] = HF_TOKEN\n",
66
- "\n",
67
- "DISCORD_WEBHOOK_URL = os.environ.get(\"DISCORD_WEBHOOK_URL\", \"\")\n",
68
- "if not DISCORD_WEBHOOK_URL:\n",
69
- " try:\n",
70
- " from google.colab import userdata\n",
71
- " DISCORD_WEBHOOK_URL = userdata.get(\"DISCORD_WEBHOOK_URL\") or \"\"\n",
72
- " except Exception:\n",
73
- " pass\n",
 
 
 
 
 
 
74
  "if DISCORD_WEBHOOK_URL:\n",
75
- " os.environ[\"DISCORD_WEBHOOK_URL\"] = DISCORD_WEBHOOK_URL.strip()\n",
76
- "print(\"Discord:\", \"SET\" if os.environ.get(\"DISCORD_WEBHOOK_URL\") else \"(none)\")\n",
77
  "\n",
78
  "from huggingface_hub import login\n",
79
- "login(token=HF_TOKEN, add_to_git_credential=True)\n",
80
- "print(\"login OK\")\n",
81
- ""
82
- ],
83
- "execution_count": null,
84
- "outputs": []
 
 
 
 
85
  },
86
  {
87
  "cell_type": "code",
88
  "metadata": {},
 
 
89
  "source": [
90
  "#@title 2. 設定\n",
91
- "\n",
92
  "REPO_ID = \"Mouserat/rat-77m\"\n",
93
  "DATASET_ID = \"Mouserat/llmjp4-2b-dataset\"\n",
94
  "BUCKET_ID = \"Mouserat/rat-77m\"\n",
@@ -96,14 +119,14 @@
96
  "SEQ_LEN = 1024\n",
97
  "VOCAB_SIZE = 64102\n",
98
  "MT5_VOCAB = 250112\n",
99
- "TARGET_ROW_L2 = 0.5 # v6実績: 行 L2=0.5(要素RMS≈0.022)。要素RMS=0.5は学習で loss≈89 に発散\n",
100
  "NOISE_DENSITY = 0.15\n",
101
  "MEAN_SPAN_LENGTH = 3.0\n",
102
  "SENTINEL_START = 64002\n",
103
- "SENTINEL_END = 64101\n",
104
  "\n",
105
- "MICRO_BATCH = 64\n",
106
- "GRAD_ACCUM = 2\n",
107
  "USE_BF16 = True\n",
108
  "USE_TF32 = True\n",
109
  "NUM_WORKERS = 4\n",
@@ -111,24 +134,28 @@
111
  "\n",
112
  "TOKEN_BUDGET = 2_000_000_000\n",
113
  "TOKENS_PER_STEP = MICRO_BATCH * GRAD_ACCUM * SEQ_LEN\n",
114
- "MAX_STEPS = TOKEN_BUDGET // TOKENS_PER_STEP\n",
115
  "WARMUP_STEPS = max(100, MAX_STEPS // 17)\n",
116
  "\n",
117
- "LR = 1e-4 # 仕様寄り・前回 3e-4 で危険だったため抑制\n",
118
  "WEIGHT_DECAY = 0.0\n",
119
- "LOG_EVERY = 50 # grad_norm 監視を細かく\n",
120
  "DISCORD_EVERY = 500\n",
121
  "SAVE_EVERY = 1000\n",
122
  "BUCKET_REPLACE_EVERY = 5000\n",
123
  "EVAL_EVERY = 1000\n",
124
  "KEEP_LAST_N = 20\n",
125
  "GRAD_NORM_ABORT = 50.0\n",
126
- "GRAD_NORM_STREAK = 2 # 旧ドキュメント運用: 連続2回\n",
127
- "UNK_RATE_ABORT = 0.02 # 再マップ後 unk 率の上限(期待 ~0.5%)\n",
128
  "USE_ADAFACTOR = True\n",
129
  "LR_SCHEDULER = \"constant_with_warmup\"\n",
130
  "WALL_CLOCK_HOURS = 20.0\n",
131
  "\n",
 
 
 
 
132
  "WORK_DIR = Path(\"/content/rat77m\") if Path(\"/content\").exists() else Path(\"./rat77m_work\")\n",
133
  "WORK_DIR.mkdir(parents=True, exist_ok=True)\n",
134
  "CKPT_DIR = WORK_DIR / \"checkpoints\"\n",
@@ -140,47 +167,36 @@
140
  " torch.backends.cuda.matmul.allow_tf32 = True\n",
141
  " torch.backends.cudnn.allow_tf32 = True\n",
142
  "\n",
143
- "print(f\"MAX_STEPS={MAX_STEPS:,} (~{MAX_STEPS*TOKENS_PER_STEP/1e9:.2f}B) warmup={WARMUP_STEPS} lr={LR} grad_abort={GRAD_NORM_STREAK}x>{GRAD_NORM_ABORT}\")\n"
144
- ],
145
- "execution_count": null,
146
- "outputs": []
147
  },
148
  {
149
  "cell_type": "code",
150
  "metadata": {},
 
 
151
  "source": [
152
  "#@title 3. Discord / Bucket ユーティリティ\n",
153
- "\n",
154
  "import urllib.request\n",
155
  "\n",
156
  "def discord_notify(content: str):\n",
157
- " \"\"\"毎回 os.environ から読む(セル1の古い空文字を掴まない)\"\"\"\n",
158
- " url = (os.environ.get(\"DISCORD_WEBHOOK_URL\") or \"\").strip()\n",
159
  " if not url:\n",
160
- " # モジュールグローバルに残っている場合のフォールバック\n",
161
- " url = (globals().get(\"DISCORD_WEBHOOK_URL\") or \"\").strip()\n",
162
- " if not url:\n",
163
- " print(\"[discord] skip: DISCORD_WEBHOOK_URL not set\")\n",
164
  " return\n",
165
  " if url.startswith(\"<\") and url.endswith(\">\"):\n",
166
  " url = url[1:-1]\n",
167
- " data = json.dumps({\n",
168
- " \"content\": content[:1900],\n",
169
- " \"allowed_mentions\": {\"parse\": []},\n",
170
- " }).encode(\"utf-8\")\n",
171
  " req = urllib.request.Request(\n",
172
- " url,\n",
173
- " data=data,\n",
174
- " headers={\n",
175
- " \"Content-Type\": \"application/json\",\n",
176
- " \"User-Agent\": \"rat77m-trainer/1.0\",\n",
177
- " \"Accept\": \"application/json\",\n",
178
- " },\n",
179
  " method=\"POST\",\n",
180
  " )\n",
181
  " try:\n",
182
  " with urllib.request.urlopen(req, timeout=15) as resp:\n",
183
  " resp.read()\n",
 
184
  " except Exception as e:\n",
185
  " body = \"\"\n",
186
  " if hasattr(e, \"read\"):\n",
@@ -189,6 +205,7 @@
189
  " except Exception:\n",
190
  " pass\n",
191
  " print(\"[discord]\", type(e).__name__, e, body)\n",
 
192
  "def ensure_bucket():\n",
193
  " try:\n",
194
  " from huggingface_hub import create_bucket\n",
@@ -208,499 +225,412 @@
208
  " try:\n",
209
  " from huggingface_hub import batch_bucket_files\n",
210
  " base = Path(local_dir)\n",
211
- " add = [(str(f), f\"{remote_prefix.strip('/')}/{f.relative_to(base).as_posix()}\") for f in base.rglob(\"*\") if f.is_file()]\n",
 
212
  " for i in range(0, len(add), 64):\n",
213
  " batch_bucket_files(BUCKET_ID, add=add[i:i+64])\n",
214
  " print(\"[bucket] python OK\", dest)\n",
215
  " return True\n",
216
  " except Exception as e:\n",
217
- " print(\"[bucket] failed\", r.stderr or e)\n",
218
  " return False\n",
219
  "\n",
220
  "ensure_bucket()\n",
221
- "discord_notify(f\"rat-77m setup · MAX_STEPS={MAX_STEPS:,} · lr={LR} · row_L2={TARGET_ROW_L2}\")\n",
222
- ""
223
- ],
224
- "execution_count": null,
225
- "outputs": []
226
  },
227
  {
228
  "cell_type": "code",
229
  "metadata": {},
 
 
230
  "source": [
231
- "#@title 4. 頻度リスト・語彙マップ構築(旧mT5 ID → 新64k ID)\n",
232
- "\n",
233
  "from huggingface_hub import hf_hub_download\n",
 
234
  "\n",
235
- "freq_path = hf_hub_download(REPO_ID, \"vocab/token_frequency_top64k.jsonl\", token=HF_TOKEN)\n",
236
- "special_path = hf_hub_download(REPO_ID, \"vocab/special_tokens.json\", token=HF_TOKEN)\n",
237
- "with open(special_path) as f:\n",
238
- " special = json.load(f)\n",
239
- "assert special[\"vocab_size\"] == VOCAB_SIZE\n",
240
- "assert special[\"extra_ids\"][\"start_id\"] == SENTINEL_START\n",
241
- "\n",
242
- "# new vocab layout\n",
243
- "token_to_new_id = {\"<pad>\": 0, \"</s>\": 1, \"<unk>\": 2}\n",
244
- "new_id_to_token = {0: \"<pad>\", 1: \"</s>\", 2: \"<unk>\"}\n",
245
- "old_id_to_new_id = {}\n",
246
  "\n",
247
- "next_id = 3\n",
248
- "with open(freq_path) as f:\n",
 
 
 
 
 
 
 
 
249
  " for line in f:\n",
250
- " item = json.loads(line)\n",
251
- " tok, old_id = item[\"token\"], int(item[\"token_id\"])\n",
252
- " if tok in (\"<pad>\", \"</s>\", \"<unk>\"):\n",
253
- " continue\n",
254
- " if next_id >= SENTINEL_START:\n",
255
  " break\n",
256
- " token_to_new_id[tok] = next_id\n",
257
- " new_id_to_token[next_id] = tok\n",
258
- " old_id_to_new_id[old_id] = next_id\n",
259
- " next_id += 1\n",
260
- "\n",
261
- "for i in range(100):\n",
262
- " tok = f\"<extra_id_{i}>\"\n",
263
- " tid = SENTINEL_START + i\n",
264
- " token_to_new_id[tok] = tid\n",
265
- " new_id_to_token[tid] = tok\n",
266
- "\n",
267
- "assert len(new_id_to_token) == VOCAB_SIZE\n",
268
- "\n",
269
- "# 高速再マップテーブル: mT5 old id -> new id(未登録は unk=2)\n",
270
- "id_remap = np.full(MT5_VOCAB, 2, dtype=np.int32)\n",
271
- "for oid, nid in old_id_to_new_id.items():\n",
272
- " if 0 <= oid < MT5_VOCAB:\n",
273
- " id_remap[oid] = nid\n",
274
- "\n",
275
- "# 特殊トークンの旧IDも可能な範囲で\n",
276
- "from transformers import AutoTokenizer\n",
277
- "mt5_tok = AutoTokenizer.from_pretrained(\"google/mt5-small\")\n",
278
  "\n",
279
  "def resolve_mt5_sentinel_ids(tokenizer, n=100):\n",
280
- " \"\"\"mT5 の <extra_id_i> を確実に取る。convert_tokens_to_ids が unk を返す環境対策。\"\"\"\n",
281
  " unk = tokenizer.unk_token_id\n",
282
- " # 1) 公式 API\n",
283
  " if hasattr(tokenizer, \"get_sentinel_token_ids\"):\n",
284
  " try:\n",
285
  " ids = list(tokenizer.get_sentinel_token_ids())\n",
286
- " if len(ids) >= n and all(i != unk and i is not None for i in ids[:n]):\n",
287
- " print(\"sentinel ids: get_sentinel_token_ids()\")\n",
288
  " return ids[:n]\n",
289
  " except Exception as e:\n",
290
- " print(\"get_sentinel_token_ids failed:\", e)\n",
291
- " # 2) convert_tokens_to_ids\n",
292
  " ids = [tokenizer.convert_tokens_to_ids(f\"<extra_id_{i}>\") for i in range(n)]\n",
293
- " if all(isinstance(i, int) and i != unk for i in ids):\n",
294
- " print(\"sentinel ids: convert_tokens_to_ids\")\n",
295
  " return ids\n",
296
- " # 3) T5/mT5 慣習: extra_id_0 が vocab 末尾\n",
297
- " # HF mT5-small は通常 vocab_size=250100、extra_id_0=250099 ...\n",
298
  " vs = int(tokenizer.vocab_size)\n",
299
  " ids = [vs - 1 - i for i in range(n)]\n",
300
- " print(f\"sentinel ids: vocab tail fallback vocab_size={vs} extra_id_0={ids[0]}\")\n",
301
- " assert all(i >= 0 for i in ids), \"sentinel fallback negative\"\n",
302
  " return ids\n",
303
  "\n",
304
- "for tok, nid in [(\"<pad>\", 0), (\"</s>\", 1), (\"<unk>\", 2)]:\n",
305
- " oid = mt5_tok.convert_tokens_to_ids(tok)\n",
306
- " if isinstance(oid, int) and 0 <= oid < MT5_VOCAB:\n",
307
- " id_remap[oid] = nid\n",
308
- "\n",
309
- "unk_oid = mt5_tok.convert_tokens_to_ids(\"<unk>\")\n",
310
- "sentinel_old_ids = resolve_mt5_sentinel_ids(mt5_tok, 100)\n",
311
- "for i, oid in enumerate(sentinel_old_ids):\n",
312
- " assert isinstance(oid, int) and oid != unk_oid and 0 <= oid < MT5_VOCAB, (\n",
313
- " f\"sentinel <extra_id_{i}> maps to unk/invalid: {oid}\"\n",
314
- " )\n",
315
  " id_remap[oid] = SENTINEL_START + i\n",
316
- "\n",
317
- "print(f\"remap table built. mapped content={len(old_id_to_new_id)} sentinels=100\")\n",
318
- "print(f\"sentinel old ids example: extra_id_0={sentinel_old_ids[0]} extra_id_99={sentinel_old_ids[99]}\")\n",
319
- "print(f\"sample: old 259 -> new {id_remap[259] if 259 < len(id_remap) else 'OOB'} (expect ~3 if 259 was ▁)\")\n",
320
- ""
321
- ],
322
- "execution_count": null,
323
- "outputs": []
324
  },
325
  {
326
  "cell_type": "code",
327
  "metadata": {},
 
 
328
  "source": [
329
- "#@title 5. mT5 emb 移植(行 L2=0.5 / v6実績)\n",
330
- "\n",
331
  "from transformers import AutoModelForSeq2SeqLM\n",
332
  "\n",
333
- "print(\"load mt5-small...\")\n",
334
  "mt5 = AutoModelForSeq2SeqLM.from_pretrained(\"google/mt5-small\", torch_dtype=torch.float32)\n",
335
- "mt5_emb = mt5.get_input_embeddings().weight.data.clone()\n",
336
- "print(f\"mT5 rowL2 mean={mt5_emb.norm(dim=1).mean().item():.4f} elementRMS mean={mt5_emb.pow(2).mean(dim=1).sqrt().mean().item():.4f}\")\n",
337
- "\n",
338
- "new_emb = torch.zeros(VOCAB_SIZE, mt5_emb.shape[1])\n",
339
- "for oid, nid in old_id_to_new_id.items():\n",
340
- " if oid < mt5_emb.shape[0]:\n",
341
- " new_emb[nid] = mt5_emb[oid]\n",
342
- "\n",
343
- "for tok, nid in [(\"<pad>\", 0), (\"</s>\", 1), (\"<unk>\", 2)]:\n",
344
- " oid = mt5_tok.convert_tokens_to_ids(tok)\n",
345
- " if isinstance(oid, int) and 0 <= oid < mt5_emb.shape[0]:\n",
346
- " new_emb[nid] = mt5_emb[oid]\n",
347
- "\n",
348
- "# セル4で作った sentinel_old_ids を再利用(無ければ再解決)\n",
349
- "if \"sentinel_old_ids\" not in globals():\n",
350
- " sentinel_old_ids = resolve_mt5_sentinel_ids(mt5_tok, 100)\n",
351
- "for i, oid in enumerate(sentinel_old_ids):\n",
352
- " assert oid != unk_oid and 0 <= oid < mt5_emb.shape[0], f\"bad sentinel oid {oid}\"\n",
353
- " new_emb[SENTINEL_START + i] = mt5_emb[oid]\n",
354
- "\n",
355
- "# v6実績: 行ごとの L2 = 0.5(要素 RMS=0.5 は大きすぎて Trainer で loss≈89 / grad 爆発)\n",
356
- "row_l2 = new_emb.norm(dim=1, keepdim=True).clamp(min=1e-8)\n",
357
- "new_emb = new_emb * (TARGET_ROW_L2 / row_l2)\n",
358
- "print(f\"after norm: rowL2 mean={new_emb.norm(dim=1).mean().item():.4f} elementRMS mean={new_emb.pow(2).mean(dim=1).sqrt().mean().item():.4f}\")\n",
359
- "\n",
 
 
 
 
 
 
360
  "del mt5, mt5_emb\n",
361
- "if torch.cuda.is_available():\n",
362
- " torch.cuda.empty_cache()\n",
363
- "print(\"emb done\")\n",
364
- ""
365
- ],
366
- "execution_count": null,
367
- "outputs": []
368
  },
369
  {
370
  "cell_type": "code",
371
  "metadata": {},
 
 
372
  "source": [
373
- "#@title 6. モデル初期化 + 健全性\n",
374
- "\n",
375
- "from transformers import T5Config, T5ForConditionalGeneration, PreTrainedTokenizerFast\n",
376
- "from tokenizers import Tokenizer\n",
377
- "from tokenizers.models import WordLevel\n",
378
  "\n",
379
  "config = T5Config(\n",
380
- " vocab_size=VOCAB_SIZE, d_model=512, d_kv=64, d_ff=2048,\n",
381
- " num_layers=6, num_decoder_layers=6, num_heads=8,\n",
382
- " relative_attention_num_buckets=32, relative_attention_max_distance=128,\n",
383
- " dropout_rate=0.1, layer_norm_epsilon=1e-6, initializer_factor=1.0,\n",
384
- " feed_forward_proj=\"relu\", is_encoder_decoder=True, use_cache=True,\n",
385
- " pad_token_id=0, eos_token_id=1, decoder_start_token_id=0, tie_word_embeddings=True,\n",
 
 
 
 
 
 
 
 
 
 
 
 
 
386
  ")\n",
387
  "model = T5ForConditionalGeneration(config)\n",
388
- "print(f\"params: {model.num_parameters():,}\")\n",
389
  "with torch.no_grad():\n",
390
  " model.get_input_embeddings().weight.copy_(new_emb)\n",
 
 
 
391
  "\n",
392
- "model.eval()\n",
393
- "with torch.no_grad():\n",
394
- " dummy = torch.randint(3, 1000, (2, 128))\n",
395
- " logits = model(input_ids=dummy, decoder_input_ids=dummy[:, :64]).logits\n",
396
- " print(f\"logits mean={logits.mean().item():.4f} std={logits.std().item():.4f}\")\n",
397
- " # 仕様レビュー: std が 5 未満が望ましい。100 超は異常\n",
398
- " if logits.std().item() >= 5:\n",
399
- " print(\"WARNING: logits std >= 5 — 監視を強めてください\")\n",
400
- " assert logits.std().item() < 100, \"logits scale abnormal\"\n",
401
- "print(f\"expected init loss ≈ {math.log(VOCAB_SIZE):.3f}\")\n",
402
- "model.train()\n",
403
- "\n",
404
- "vocab = {tok: idx for idx, tok in new_id_to_token.items()}\n",
405
- "tok_backend = Tokenizer(WordLevel(vocab=vocab, unk_token=\"<unk>\"))\n",
406
- "tok_backend.add_special_tokens([\"<pad>\", \"</s>\", \"<unk>\"] + [f\"<extra_id_{i}>\" for i in range(100)])\n",
407
- "fast_tok = PreTrainedTokenizerFast(\n",
408
- " tokenizer_object=tok_backend, eos_token=\"</s>\", unk_token=\"<unk>\",\n",
409
- " pad_token=\"<pad>\", model_max_length=SEQ_LEN,\n",
410
- ")\n",
411
- "fast_tok.add_special_tokens({\"additional_special_tokens\": [f\"<extra_id_{i}>\" for i in range(100)]})\n",
412
- "print(\"tokenizer ready\")\n",
413
- ""
414
- ],
415
- "execution_count": null,
416
- "outputs": []
417
  },
418
  {
419
  "cell_type": "code",
420
  "metadata": {},
421
- "source": [
422
- "#@title 7. init-fixed 保存\n",
423
- "\n",
424
- "from huggingface_hub import HfApi\n",
425
- "init_dir = WORK_DIR / \"init-fixed\"\n",
426
- "init_dir.mkdir(exist_ok=True)\n",
427
- "model.save_pretrained(init_dir)\n",
428
- "fast_tok.save_pretrained(init_dir)\n",
429
- "# 再マップ表も保存(再現用)\n",
430
- "np.save(init_dir / \"id_remap_mt5_to_64k.npy\", id_remap)\n",
431
- "cfg = json.loads((init_dir / \"config.json\").read_text())\n",
432
- "cfg[\"max_length\"] = SEQ_LEN\n",
433
- "(init_dir / \"config.json\").write_text(json.dumps(cfg, indent=2))\n",
434
- "HfApi().upload_folder(folder_path=str(init_dir), repo_id=REPO_ID, path_in_repo=\"init-fixed\",\n",
435
- " token=HF_TOKEN, commit_message=\"init-fixed elementRMS=0.5 + id_remap\")\n",
436
- "print(\"init-fixed pushed\")\n",
437
- ""
438
- ],
439
  "execution_count": null,
440
- "outputs": []
 
 
 
 
 
 
 
 
 
441
  },
442
  {
443
  "cell_type": "code",
444
  "metadata": {},
445
- "source": [
446
- "#@title 8. データダウンロード + ★旧ID確認 + 再マップ前提チェック\n",
447
- "\n",
448
- "from huggingface_hub import list_repo_files, hf_hub_download\n",
449
- "\n",
450
- "bin_files = [f for f in list_repo_files(DATASET_ID, repo_type=\"dataset\", token=HF_TOKEN) if f.endswith(\".bin\")]\n",
451
- "local_bins = []\n",
452
- "ja_tokens = en_tokens = 0\n",
453
- "for fname in bin_files:\n",
454
- " p = hf_hub_download(DATASET_ID, fname, repo_type=\"dataset\", token=HF_TOKEN, local_dir=str(DATA_DIR))\n",
455
- " local_bins.append(p)\n",
456
- " n = Path(p).stat().st_size // 4\n",
457
- " if Path(fname).name.startswith(\"ja_\"):\n",
458
- " ja_tokens += n\n",
459
- " elif Path(fname).name.startswith(\"en_\"):\n",
460
- " en_tokens += n\n",
461
- " print(f\" {fname}: {Path(p).stat().st_size//1024**2} MB\")\n",
462
- "print(f\"ja={ja_tokens/1e9:.3f}B en={en_tokens/1e9:.3f}B\")\n",
463
- "\n",
464
- "# ★最重要診断: 最頻 ID が 259(mT5 ▁) なら未変換データ\n",
465
- "arr = np.memmap(local_bins[0], dtype=\"<u4\", mode=\"r\")\n",
466
- "sample = np.asarray(arr[:10_000_000], dtype=np.int64)\n",
467
- "# bincount は max が大きいと重いのでユニークで近似\n",
468
- "uniq, counts = np.unique(sample, return_counts=True)\n",
469
- "top_old = int(uniq[counts.argmax()])\n",
470
- "print(f\"最頻旧ID: {top_old} (mT5の▁なら 259 付近 / 既に新IDなら 3 付近)\")\n",
471
- "if top_old == 259 or top_old > VOCAB_SIZE:\n",
472
- " print(\"→ .bin は mT5 旧 ID。学習時に id_remap で変換します(必須)\")\n",
473
- "elif top_old < 10:\n",
474
- " print(\"→ すでに新 ID っぽい。remap を恒等に近い扱いでもよいが、表適用は安全側で継続\")\n",
475
- "discord_notify(f\"data top_id={top_old} (259=mT5 raw, ~3=already new)\")\n",
476
- ""
477
- ],
478
  "execution_count": null,
479
- "outputs": []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
480
  },
481
  {
482
  "cell_type": "code",
483
  "metadata": {},
 
 
484
  "source": [
485
- "#@title 9. Dataset(再マップ適用)/ Collator / train-eval 分割\n",
486
- "\n",
487
  "from torch.utils.data import Dataset, Subset\n",
488
  "\n",
489
- "class BinTokenDataset(Dataset):\n",
490
- " \"\"\"uint32 LE .bin (mT5 IDs) → id_remap → 新64k IDs、長さ SEQ_LEN\"\"\"\n",
491
- " def __init__(self, bin_paths, id_remap, seq_len=1024):\n",
492
  " self.seq_len = seq_len\n",
493
- " self.id_remap = id_remap\n",
494
- " self.shards = []\n",
495
- " self.cum = [0]\n",
496
- " for p in bin_paths:\n",
497
- " arr = np.memmap(p, dtype=\"<u4\", mode=\"r\")\n",
498
- " n = len(arr) // seq_len\n",
499
- " if n > 0:\n",
500
- " self.shards.append(arr)\n",
501
- " self.cum.append(self.cum[-1] + n)\n",
502
- " self.total = self.cum[-1]\n",
503
- " print(f\"Dataset sequences: {self.total:,}\")\n",
504
  "\n",
505
  " def __len__(self):\n",
506
- " return self.total\n",
507
- "\n",
508
- " def __getitem__(self, idx):\n",
509
- " for i in range(len(self.shards)):\n",
510
- " if idx < self.cum[i + 1]:\n",
511
- " local = idx - self.cum[i]\n",
512
- " arr = self.shards[i]\n",
513
- " start = local * self.seq_len\n",
514
- " old = np.asarray(arr[start:start + self.seq_len], dtype=np.int64)\n",
515
- " # 範囲外・負値は unk(2)。clip して末尾 ID に寄せない\n",
516
- " valid = (old >= 0) & (old < len(self.id_remap))\n",
517
- " new = np.full(old.shape, 2, dtype=np.int64)\n",
518
- " new[valid] = self.id_remap[old[valid]]\n",
519
- " return {\"input_ids\": torch.from_numpy(new)}\n",
520
- " raise IndexError(idx)\n",
521
- "\n",
522
- "def span_corrupt(input_ids, noise_density=0.15, mean_span_length=3.0,\n",
523
- " sentinel_start=64002, sentinel_end=64101, seq_len=1024, rng=None):\n",
524
- " if rng is None:\n",
525
- " rng = np.random.default_rng()\n",
526
- " length = input_ids.size(0)\n",
527
- " num_noise = min(max(int(round(length * noise_density)), 1), length - 1)\n",
528
- " num_spans = max(int(round(num_noise / mean_span_length)), 1)\n",
529
- " noise_mask = np.zeros(length, dtype=bool)\n",
530
- " for _ in range(num_spans):\n",
531
- " span_len = max(1, int(rng.exponential(mean_span_length)))\n",
532
- " start = int(rng.integers(0, max(1, length - span_len + 1)))\n",
533
- " noise_mask[start:start + span_len] = True\n",
534
- " new_input, new_target = [], []\n",
535
- " sid = sentinel_start\n",
 
 
 
 
 
 
 
 
 
 
 
 
 
536
  " i = 0\n",
537
- " while i < length:\n",
538
- " if noise_mask[i]:\n",
539
- " new_input.append(sid)\n",
540
- " new_target.append(sid)\n",
541
- " while i < length and noise_mask[i]:\n",
542
- " new_target.append(int(input_ids[i]))\n",
543
- " i += 1\n",
544
- " sid += 1\n",
545
- " if sid > sentinel_end:\n",
546
- " sid = sentinel_start\n",
547
- " else:\n",
548
- " new_input.append(int(input_ids[i]))\n",
 
 
 
 
549
  " i += 1\n",
550
- " new_target.append(1)\n",
551
- " def fit(x, max_len, pad):\n",
552
- " x = x[:max_len]\n",
553
- " return x + [pad] * (max_len - len(x))\n",
554
- " input_out = torch.tensor(fit(new_input, seq_len, 0), dtype=torch.long)\n",
555
- " labels = torch.tensor(fit(new_target, seq_len, -100), dtype=torch.long)\n",
556
- " attention_mask = (input_out != 0).long()\n",
557
- " return input_out, attention_mask, labels\n",
 
558
  "\n",
559
  "class SpanDataCollator:\n",
560
- " def __init__(self):\n",
561
- " self._rng = None\n",
562
- " def _get_rng(self):\n",
563
- " if self._rng is None:\n",
564
- " self._rng = np.random.default_rng((int(time.time()*1e6) + os.getpid()) % (2**32-1))\n",
565
- " return self._rng\n",
566
  " def __call__(self, features):\n",
567
- " inputs, masks, labels = [], [], []\n",
568
- " rng = self._get_rng()\n",
569
  " for f in features:\n",
570
- " ids = f[\"input_ids\"] if isinstance(f, dict) else f\n",
571
- " if not torch.is_tensor(ids):\n",
572
- " ids = torch.tensor(ids, dtype=torch.long)\n",
573
- " inp, mask, lab = span_corrupt(ids, NOISE_DENSITY, MEAN_SPAN_LENGTH,\n",
574
- " SENTINEL_START, SENTINEL_END, SEQ_LEN, rng=rng)\n",
575
- " inputs.append(inp); masks.append(mask); labels.append(lab)\n",
576
- " return {\"input_ids\": torch.stack(inputs), \"attention_mask\": torch.stack(masks), \"labels\": torch.stack(labels)}\n",
577
- "\n",
578
- "full_ds = BinTokenDataset(local_bins, id_remap, seq_len=SEQ_LEN)\n",
579
- "\n",
580
- "# 再マップ後確認: ランダム位置からサンプリング(先頭偏り回避)\n",
581
- "rng_chk = np.random.default_rng(42)\n",
582
- "n_seq = len(full_ds)\n",
583
- "sample_idx = rng_chk.choice(n_seq, size=min(2000, n_seq), replace=False)\n",
584
- "sample_new = np.concatenate([full_ds[int(i)][\"input_ids\"].numpy() for i in sample_idx])\n",
585
- "top_new = int(np.bincount(sample_new).argmax())\n",
586
- "print(f\"再マップ後 最頻新ID: {top_new} (▁ なら 3 付近が��然)\")\n",
587
- "assert int(sample_new.max()) < VOCAB_SIZE\n",
588
- "unk_rate = float((sample_new == 2).mean())\n",
589
- "print(f\"unk rate (random sample)={unk_rate:.4%}\")\n",
590
- "UNK_LIMIT = globals().get(\"UNK_RATE_ABORT\", 0.02)\n",
591
- "if unk_rate > UNK_LIMIT:\n",
592
- " raise RuntimeError(f\"unk rate {unk_rate:.4%} > {UNK_LIMIT:.2%} after remap\")\n",
593
- "discord_notify(f\"remap ok top_new_id={top_new} unk_rate={unk_rate:.4%}\")\n",
594
- "\n",
595
- "# train/eval: シャッフルして分割(末尾だけ言語偏りになるのを避ける)\n",
596
- "n_eval = max(1, int(n_seq * 0.015))\n",
597
- "perm = rng_chk.permutation(n_seq)\n",
598
- "eval_idx = perm[:n_eval].tolist()\n",
599
- "train_idx = perm[n_eval:].tolist()\n",
600
- "train_ds = Subset(full_ds, train_idx)\n",
601
- "eval_ds = Subset(full_ds, eval_idx)\n",
602
- "print(f\"train={len(train_ds):,} eval={len(eval_ds):,} (shuffled split)\")\n",
603
- "\n",
604
- "# eval 用: span corruption の RNG を固定して step 間比較可能に\n",
605
- "class EvalSpanCollator(SpanDataCollator):\n",
606
- " def __init__(self, seed=12345):\n",
607
- " self._rng = np.random.default_rng(seed)\n",
608
- " def _get_rng(self):\n",
609
- " # 毎回同じ系列を再現するため、call ごとに seed から作り直さず固定 RNG を進める\n",
610
- " # 比較安定化のため、features の input ハッシュで seed を決める\n",
611
- " return self._rng\n",
612
  " def __call__(self, features):\n",
613
- " inputs, masks, labels = [], [], []\n",
614
- " for f in features:\n",
615
- " ids = f[\"input_ids\"] if isinstance(f, dict) else f\n",
616
- " if not torch.is_tensor(ids):\n",
617
- " ids = torch.tensor(ids, dtype=torch.long)\n",
618
- " # サンプルごとに決定的 seed\n",
619
- " seed = int(ids[:32].sum().item()) % (2**31 - 1)\n",
620
- " rng = np.random.default_rng(seed)\n",
621
- " inp, mask, lab = span_corrupt(\n",
622
- " ids, NOISE_DENSITY, MEAN_SPAN_LENGTH,\n",
623
- " SENTINEL_START, SENTINEL_END, SEQ_LEN, rng=rng,\n",
624
- " )\n",
625
- " inputs.append(inp); masks.append(mask); labels.append(lab)\n",
 
626
  " return {\n",
627
- " \"input_ids\": torch.stack(inputs),\n",
628
- " \"attention_mask\": torch.stack(masks),\n",
629
- " \"labels\": torch.stack(labels),\n",
630
  " }\n",
631
  "\n",
632
- "eval_collator = EvalSpanCollator()\n",
633
- "print(\"eval collator: per-sample deterministic mask\")\n",
634
- ""
635
- ],
636
- "execution_count": null,
637
- "outputs": []
638
  },
639
  {
640
  "cell_type": "code",
641
  "metadata": {},
 
 
642
  "source": [
643
- "#@title 10. 学習前診断(1 step)\n",
644
- "\n",
645
- "# 前回 v3 は step1 loss=770 で即異常だった。ここで潰す\n",
646
- "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
647
- "model = model.to(device)\n",
648
  "model.train()\n",
649
  "collator = SpanDataCollator()\n",
650
  "batch = collator([train_ds[i] for i in range(min(8, len(train_ds)))])\n",
651
  "batch = {k: v.to(device) for k, v in batch.items()}\n",
652
  "\n",
 
 
 
 
 
653
  "model.zero_grad(set_to_none=True)\n",
654
- "with torch.autocast(device_type=\"cuda\", dtype=torch.bfloat16, enabled=(device.type==\"cuda\" and USE_BF16)):\n",
655
- " out = model(**batch)\n",
656
- " loss = out.loss\n",
657
- "loss.backward()\n",
658
- "gn = torch.nn.utils.clip_grad_norm_(model.parameters(), 1e9).item()\n",
659
  "model.zero_grad(set_to_none=True)\n",
660
  "\n",
661
- "print(f\"diag loss={loss.item():.4f} (expect ~{math.log(VOCAB_SIZE):.2f})\")\n",
662
- "print(f\"diag grad_norm={gn:.4f} (expect << 50)\")\n",
663
- "logits_std = out.logits.float().std().item()\n",
664
- "print(f\"diag logits_std={logits_std:.4f} (prefer < 5)\")\n",
665
- "\n",
666
- "ok = True\n",
667
- "if not (8.0 <= loss.item() <= 14.0):\n",
668
- " print(\"FAIL: loss が ln(V) 近辺にない\")\n",
669
- " ok = False\n",
670
- "if gn >= 50:\n",
671
- " print(\"FAIL: grad_norm >= 50\")\n",
672
- " ok = False\n",
673
- "if logits_std >= 100:\n",
674
- " print(\"FAIL: logits std 異常\")\n",
675
- " ok = False\n",
676
  "if not ok:\n",
677
- " discord_notify(f\"PRE-TRAIN DIAG FAIL loss={loss.item():.3f} gn={gn:.2f} logits_std={logits_std:.2f}\")\n",
678
- " raise RuntimeError(\"Pre-train diagnostic failed — 学習に入らない\")\n",
 
 
679
  "print(\"PRE-TRAIN DIAG OK\")\n",
680
- "discord_notify(f\"PRE-TRAIN DIAG OK loss={loss.item():.3f} gn={gn:.2f} logits_std={logits_std:.2f}\")\n",
681
- ""
682
- ],
683
- "execution_count": null,
684
- "outputs": []
685
  },
686
  {
687
  "cell_type": "code",
688
  "metadata": {},
 
 
689
  "source": [
690
  "#@title 11. Trainer + Discord + Bucket + 連続 grad 停止 + 再開\n",
691
- "\n",
692
- "from transformers import Trainer, TrainingArguments, TrainerCallback, TrainerState, TrainerControl, Adafactor\n",
693
- "\n",
694
  "\n",
695
  "class EvalSwapTrainer(Trainer):\n",
696
- " \"\"\"eval 時だけ data_collator を決定的マスク版に差し替える\"\"\"\n",
697
  " def __init__(self, *args, eval_data_collator=None, **kwargs):\n",
698
  " super().__init__(*args, **kwargs)\n",
699
- " self._train_collator = self.data_collator\n",
700
  " self._eval_collator = eval_data_collator or self.data_collator\n",
701
  "\n",
702
  " def get_eval_dataloader(self, eval_dataset=None):\n",
703
- " # evaluate() 経路で eval collator を使う\n",
704
  " original = self.data_collator\n",
705
  " self.data_collator = self._eval_collator\n",
706
  " try:\n",
@@ -718,52 +648,38 @@
718
  " self.t0 = time.time()\n",
719
  " self.last = 0\n",
720
  " self.warmup_alerted = set()\n",
 
 
721
  " def on_log(self, args, state, control, logs=None, **kwargs):\n",
722
  " if not logs or state.global_step <= 0:\n",
723
  " return control\n",
724
  " step = state.global_step\n",
725
- " loss = logs.get(\"loss\")\n",
726
- " gn = logs.get(\"grad_norm\")\n",
727
- " lr = logs.get(\"learning_rate\")\n",
728
- " # soft loss warning: 初回超過 + 500stepごとだけ(連投防止)\n",
729
  " if loss is not None and float(loss) > 35:\n",
730
- " if not hasattr(self, \"_loss_high_since\"):\n",
731
- " self._loss_high_since = None\n",
732
  " if self._loss_high_since is None:\n",
733
  " self._loss_high_since = step\n",
734
- " wmsg = f\"⚠️ **loss high (first)** step `{step}` loss=`{loss}` grad_norm=`{gn}` lr=`{lr}`\"\n",
735
- " print(\"[warn]\", wmsg)\n",
736
- " discord_notify(wmsg)\n",
737
  " elif step % 500 == 0:\n",
738
- " wmsg = f\"⚠️ **loss still high** step `{step}` loss=`{loss}` grad_norm=`{gn}` lr=`{lr}`\"\n",
739
- " print(\"[warn]\", wmsg)\n",
740
- " discord_notify(wmsg)\n",
741
  " else:\n",
742
  " self._loss_high_since = None\n",
743
- " # warmup 終了付近を特に通知 (0.8x / 1.0x / 1.2x)\n",
744
  " for mult, tag in [(0.8, \"warmup-80%\"), (1.0, \"warmup-END\"), (1.2, \"post-warmup+20%\")]:\n",
745
  " mark = int(self.warmup_steps * mult)\n",
746
  " if mark > 0 and step >= mark and tag not in self.warmup_alerted:\n",
747
  " self.warmup_alerted.add(tag)\n",
748
- " msg = (\n",
749
- " f\"⚠️ **{tag}** step `{step}` (warmup={self.warmup_steps})\\n\"\n",
750
- " f\"loss=`{loss}` lr=`{lr}` grad_norm=`{gn}`\\n\"\n",
751
- " f\"ここが前回いちばん危険だった帯です\"\n",
752
- " )\n",
753
- " print(\"[discord-warmup]\", msg.replace(\"\\n\", \" | \"))\n",
754
- " discord_notify(msg)\n",
755
  " if step % self.every != 0 or step == self.last:\n",
756
  " return control\n",
757
  " self.last = step\n",
758
  " tok = step * TOKENS_PER_STEP\n",
759
- " msg = (\n",
760
  " f\"**rat-77m** step `{step}/{MAX_STEPS}`\\n\"\n",
761
- " f\"loss=`{loss}` eval_loss=`{logs.get('eval_loss')}` \"\n",
762
- " f\"lr=`{lr}` grad_norm=`{gn}`\\n\"\n",
763
- " f\"tokens≈`{tok/1e9:.3f}B` elapsed=`{(time.time()-self.t0)/3600:.2f}h`\"\n",
764
  " )\n",
765
- " print(\"[discord]\", msg.replace(\"\\n\", \" | \"))\n",
766
- " discord_notify(msg)\n",
767
  " return control\n",
768
  "\n",
769
  "class BucketReplaceCallback(TrainerCallback):\n",
@@ -776,16 +692,14 @@
776
  " return control\n",
777
  " latest = ckpts[-1]\n",
778
  " bucket_sync_dir(str(latest), f\"{BUCKET_RUN}/{latest.name}\")\n",
779
- " if state.global_step % self.every == 0:\n",
780
  " bucket_sync_dir(str(latest), BUCKET_LATEST)\n",
781
  " discord_notify(f\"Bucket latest @ {state.global_step}: {latest.name}\")\n",
782
  " return control\n",
783
  "\n",
784
  "class GradNormStreakCallback(TrainerCallback):\n",
785
- " def __init__(self, thr=50.0, streak=3):\n",
786
- " self.thr = thr\n",
787
- " self.streak_need = streak\n",
788
- " self.streak = 0\n",
789
  " def on_log(self, args, state, control, logs=None, **kwargs):\n",
790
  " if not logs or logs.get(\"grad_norm\") is None:\n",
791
  " return control\n",
@@ -793,11 +707,8 @@
793
  " if gn > self.thr:\n",
794
  " self.streak += 1\n",
795
  " print(f\"grad_norm spike {gn:.2f} streak={self.streak}/{self.streak_need}\")\n",
796
- " # 通知は streak 開始時と abort 時のみ(10stepごとの連投を避ける)\n",
797
  " if self.streak == 1 or self.streak >= self.streak_need:\n",
798
- " discord_notify(\n",
799
- " f\"⚠️ **grad_norm spike** `{gn:.2f}` streak={self.streak}/{self.streak_need} step `{state.global_step}`\"\n",
800
- " )\n",
801
  " else:\n",
802
  " self.streak = 0\n",
803
  " if self.streak >= self.streak_need:\n",
@@ -809,34 +720,27 @@
809
  "\n",
810
  "class WallClockStopCallback(TrainerCallback):\n",
811
  " def __init__(self, hours=20.0):\n",
812
- " self.limit = hours * 3600\n",
813
- " self.t0 = time.time()\n",
814
  " def on_step_end(self, args, state, control, **kwargs):\n",
815
  " if time.time() - self.t0 >= self.limit:\n",
816
- " print(\"Wall clock stop\")\n",
817
  " discord_notify(f\"Wall clock stop step={state.global_step}\")\n",
818
  " control.should_training_stop = True\n",
819
  " control.should_save = True\n",
820
  " return control\n",
821
  "\n",
822
  "def try_resume_from_bucket():\n",
823
- " \"\"\"latest / run01 / ローカルから、step 番号が最大の checkpoint を選ぶ\"\"\"\n",
824
  " local = CKPT_DIR / \"resume\"\n",
825
  " local.mkdir(exist_ok=True)\n",
826
  " candidates = []\n",
827
  " for prefix in [BUCKET_LATEST, BUCKET_RUN]:\n",
828
  " dest = Path(local) / Path(prefix).name.replace(\"/\", \"_\")\n",
829
- " r = subprocess.run(\n",
830
- " [\"hf\", \"buckets\", \"sync\", f\"hf://buckets/{BUCKET_ID}/{prefix}\", str(dest)],\n",
831
- " capture_output=True, text=True,\n",
832
- " )\n",
833
  " if r.returncode != 0:\n",
834
  " continue\n",
835
  " for p in dest.rglob(\"checkpoint-*\"):\n",
836
  " if p.is_dir() and p.name.split(\"-\")[-1].isdigit():\n",
837
  " candidates.append((int(p.name.split(\"-\")[-1]), p))\n",
838
  " if (dest / \"config.json\").exists():\n",
839
- " # flat latest without checkpoint- prefix → step 0 扱い\n",
840
  " candidates.append((0, dest))\n",
841
  " for p in CKPT_DIR.glob(\"checkpoint-*\"):\n",
842
  " if p.is_dir() and p.name.split(\"-\")[-1].isdigit():\n",
@@ -851,28 +755,15 @@
851
  "resume_path = try_resume_from_bucket()\n",
852
  "print(\"resume_path:\", resume_path)\n",
853
  "\n",
854
- "import inspect\n",
855
  "_ta_kwargs = dict(\n",
856
- " output_dir=str(CKPT_DIR),\n",
857
- " max_steps=MAX_STEPS,\n",
858
- " per_device_train_batch_size=MICRO_BATCH,\n",
859
- " gradient_accumulation_steps=GRAD_ACCUM,\n",
860
- " learning_rate=LR,\n",
861
- " weight_decay=WEIGHT_DECAY,\n",
862
- " warmup_steps=WARMUP_STEPS,\n",
863
- " lr_scheduler_type=LR_SCHEDULER,\n",
864
- " logging_steps=LOG_EVERY,\n",
865
- " eval_steps=EVAL_EVERY,\n",
866
- " save_strategy=\"steps\",\n",
867
- " save_steps=SAVE_EVERY,\n",
868
- " save_total_limit=KEEP_LAST_N,\n",
869
- " bf16=USE_BF16 and device.type == \"cuda\",\n",
870
- " dataloader_num_workers=NUM_WORKERS,\n",
871
- " dataloader_pin_memory=PIN_MEMORY,\n",
872
- " remove_unused_columns=False,\n",
873
- " report_to=\"none\",\n",
874
- " max_grad_norm=1.0,\n",
875
- " disable_tqdm=False,\n",
876
  ")\n",
877
  "_sig = inspect.signature(TrainingArguments.__init__)\n",
878
  "if \"eval_strategy\" in _sig.parameters:\n",
@@ -885,8 +776,11 @@
885
  " _ta_kwargs[\"tf32\"] = USE_TF32\n",
886
  "if \"optim\" in _sig.parameters:\n",
887
  " _ta_kwargs[\"optim\"] = \"adafactor\" if USE_ADAFACTOR else \"adamw_torch\"\n",
 
 
 
888
  "training_args = TrainingArguments(**_ta_kwargs)\n",
889
- "print(\"TrainingArguments:\", \"transformers\", __import__(\"transformers\").__version__, \"keys\", sorted(_ta_kwargs.keys()))\n",
890
  "\n",
891
  "optimizer = None\n",
892
  "if USE_ADAFACTOR:\n",
@@ -897,10 +791,7 @@
897
  " )\n",
898
  "\n",
899
  "trainer_kw = dict(\n",
900
- " model=model,\n",
901
- " args=training_args,\n",
902
- " train_dataset=train_ds,\n",
903
- " eval_dataset=eval_ds,\n",
904
  " data_collator=SpanDataCollator(),\n",
905
  " callbacks=[\n",
906
  " DiscordCallback(DISCORD_EVERY, warmup_steps=WARMUP_STEPS),\n",
@@ -911,122 +802,43 @@
911
  ")\n",
912
  "if optimizer is not None:\n",
913
  " trainer_kw[\"optimizers\"] = (optimizer, None)\n",
914
- "\n",
915
  "trainer_kw[\"eval_data_collator\"] = EvalSpanCollator()\n",
916
  "trainer = EvalSwapTrainer(**trainer_kw)\n",
917
- "try:\n",
918
- " from transformers.trainer_callback import PrinterCallback\n",
919
- " # trainer.remove_callback(PrinterCallback) # keep for progress loss lines\n",
920
- " print(\"PrinterCallback kept (loss lines on bar)\")\n",
921
- "except Exception as e:\n",
922
- " print(\"PrinterCallback remove:\", e)\n",
923
- "\n",
924
- "# 開始時に学習設定をすべて Discord へ\n",
925
- "_settings = {\n",
926
- " \"REPO_ID\": REPO_ID,\n",
927
- " \"DATASET_ID\": DATASET_ID,\n",
928
- " \"BUCKET_ID\": BUCKET_ID,\n",
929
- " \"SEQ_LEN\": SEQ_LEN,\n",
930
- " \"VOCAB_SIZE\": VOCAB_SIZE,\n",
931
- " \"TARGET_ROW_L2\": TARGET_ROW_L2,\n",
932
- " \"MICRO_BATCH\": MICRO_BATCH,\n",
933
- " \"GRAD_ACCUM\": GRAD_ACCUM,\n",
934
- " \"TOKENS_PER_STEP\": TOKENS_PER_STEP,\n",
935
- " \"TOKEN_BUDGET\": TOKEN_BUDGET,\n",
936
- " \"MAX_STEPS\": MAX_STEPS,\n",
937
- " \"WARMUP_STEPS\": WARMUP_STEPS,\n",
938
- " \"LR\": LR,\n",
939
- " \"LR_SCHEDULER\": LR_SCHEDULER,\n",
940
- " \"USE_ADAFACTOR\": USE_ADAFACTOR,\n",
941
- " \"USE_BF16\": USE_BF16,\n",
942
- " \"USE_TF32\": USE_TF32,\n",
943
- " \"LOG_EVERY\": LOG_EVERY,\n",
944
- " \"DISCORD_EVERY\": DISCORD_EVERY,\n",
945
- " \"SAVE_EVERY\": SAVE_EVERY,\n",
946
- " \"EVAL_EVERY\": EVAL_EVERY,\n",
947
- " \"BUCKET_REPLACE_EVERY\": BUCKET_REPLACE_EVERY,\n",
948
- " \"KEEP_LAST_N\": KEEP_LAST_N,\n",
949
- " \"GRAD_NORM_ABORT\": GRAD_NORM_ABORT,\n",
950
- " \"GRAD_NORM_STREAK\": GRAD_NORM_STREAK,\n",
951
- " \"UNK_RATE_ABORT\": globals().get(\"UNK_RATE_ABORT\", 0.02),\n",
952
- " \"WALL_CLOCK_HOURS\": WALL_CLOCK_HOURS,\n",
953
- " \"NOISE_DENSITY\": NOISE_DENSITY,\n",
954
- " \"MEAN_SPAN_LENGTH\": MEAN_SPAN_LENGTH,\n",
955
- " \"resume_path\": resume_path,\n",
956
- " \"train_size\": len(train_ds),\n",
957
- " \"eval_size\": len(eval_ds),\n",
958
- " \"device\": str(device),\n",
959
- "}\n",
960
- "_settings_txt = \"\\n\".join(f\"{k}={v}\" for k, v in _settings.items())\n",
961
- "print(\"=== TRAIN SETTINGS ===\")\n",
962
- "print(_settings_txt)\n",
963
- "# Discord は 2000 文字制限があるので分割\n",
964
- "_head = f\"**rat-77m TRAIN START**\\n```\\n{_settings_txt}\\n```\"\n",
965
- "if len(_head) <= 1900:\n",
966
- " discord_notify(_head)\n",
967
- "else:\n",
968
- " discord_notify(\"**rat-77m TRAIN START** (settings part 1)\")\n",
969
- " for i in range(0, len(_settings_txt), 1700):\n",
970
- " discord_notify(f\"```\\n{_settings_txt[i:i+1700]}\\n```\")\n",
971
- "\n",
972
- "discord_notify(f\"Training start steps={MAX_STEPS} lr={LR} resume={resume_path if resume_path else 'none'}\")\n",
973
  "train_result = trainer.train(resume_from_checkpoint=resume_path)\n",
974
  "print(train_result)\n",
975
  "discord_notify(f\"Training finished {train_result.metrics}\")\n"
976
- ],
977
- "execution_count": null,
978
- "outputs": []
979
  },
980
  {
981
  "cell_type": "code",
982
  "metadata": {},
 
 
983
  "source": [
984
- "#@title 12. 完了: final-20 を Bucket へ\n",
985
- "\n",
986
- "from huggingface_hub import HfApi\n",
987
- "\n",
988
  "ckpts = sorted([p for p in CKPT_DIR.glob(\"checkpoint-*\") if p.is_dir()],\n",
989
- " key=lambda p: int(p.name.split(\"-\")[-1]))[-KEEP_LAST_N:]\n",
990
- "print(f\"upload last {len(ckpts)} to final-20/\")\n",
991
- "for ck in ckpts:\n",
992
- " bucket_sync_dir(str(ck), f\"final-20/{ck.name}\")\n",
 
993
  "\n",
994
  "final_dir = WORK_DIR / \"final\"\n",
995
  "final_dir.mkdir(exist_ok=True)\n",
996
  "trainer.save_model(str(final_dir))\n",
997
- "fast_tok.save_pretrained(final_dir)\n",
998
- "np.save(final_dir / \"id_remap_mt5_to_64k.npy\", id_remap)\n",
999
  "bucket_sync_dir(str(final_dir), \"final\")\n",
1000
- "bucket_sync_dir(str(final_dir), BUCKET_LATEST)\n",
1001
- "\n",
1002
- "HfApi().upload_folder(folder_path=str(final_dir), repo_id=REPO_ID, path_in_repo=\".\",\n",
1003
- " token=HF_TOKEN, commit_message=\"Final model\")\n",
1004
- "discord_notify(f\"**rat-77m DONE** final-20={len(ckpts)} https://huggingface.co/{REPO_ID}\")\n",
1005
- "print(\"done\")\n",
1006
- ""
1007
- ],
1008
- "execution_count": null,
1009
- "outputs": []
1010
- },
1011
- {
1012
- "cell_type": "markdown",
1013
- "metadata": {},
1014
- "source": [
1015
- "## チェックリスト(学習開始前)\n",
1016
- "\n",
1017
- "1. セル8: 最頻旧ID が 259 なら生 mT5 → セル9で remap される\n",
1018
- "2. セル9: 再マップ後 最頻が 3 付近、unk_rate が小さい\n",
1019
- "3. セル10: loss≈11 / grad_norm≪50 / logits_std が極端でない\n",
1020
- "4. `DISCORD_WEBHOOK_URL` 設定済みか\n",
1021
- "5. Bucket `Mouserat/rat-77m` が作成済みか(セル3)\n",
1022
- "\n",
1023
- "## 仕様との対応\n",
1024
- "- **行 L2=0.5**(v6実績。要素RMS=0.5は発散)← 以前の「行 L2=0.5」は修正済み\n",
1025
- "- LR=1e-4 / constant_with_warmup\n",
1026
- "- grad_norm>50 が **連続2回**で停止\n",
1027
- "- sentinel は unk と異なる ID であることの assert\n",
1028
- "\n",
1029
- ""
1030
  ]
1031
  }
1032
  ]
 
1
  {
2
  "nbformat": 4,
3
+ "nbformat_minor": 5,
4
  "metadata": {
5
  "kernelspec": {
6
+ "display_name": "Python 3",
7
+ "language": "python",
8
+ "name": "python3"
9
  },
10
  "language_info": {
11
  "name": "python"
12
  },
13
+ "colab": {
14
+ "provenance": []
15
+ }
16
  },
17
  "cells": [
18
  {
19
  "cell_type": "markdown",
20
  "metadata": {},
21
  "source": [
22
+ "# rat-77m 事前学習ノート(完全版)\n",
23
+ "\n",
24
+ "**Mouserat/rat-77m** · T5 日英 · vocab 64102 · seq 1024 · ~77M\n",
25
+ "\n",
26
+ "### 再発防止\n",
27
+ "1. emb は **行 L2=0.5** で移植\n",
28
+ "2. `.bin` は mT5 旧 ID → **再マップ必須**\n",
29
+ "3. 学習前診断が範囲外なら **raise で停止**\n",
30
+ "4. grad_norm>50 連続2回で ABORT\n",
31
+ "5. Discord / Bucket / 進捗バー\n",
32
+ "\n",
33
+ "セル10が OK になるまでセル11に入らないこと。\n"
34
+ ]
35
+ },
36
+ {
37
+ "cell_type": "code",
38
+ "metadata": {},
39
+ "outputs": [],
40
+ "execution_count": null,
41
+ "source": [
42
+ "#@title 0. チェックリスト\n",
43
+ "print(\"\"\"\n",
44
+ "[ ] HF_TOKEN / DISCORD_WEBHOOK_URL\n",
45
+ "[ ] セル5で行 L2 mean ≈ 0.5\n",
46
+ "[ ] セル9で unk_rate < 2%\n",
47
+ "[ ] セル10: loss 8〜16 / grad_norm < 50 / logits_std < 5\n",
48
+ "[ ] だめならセル11に進まない\n",
49
+ "\"\"\")\n"
50
  ]
51
  },
52
  {
53
  "cell_type": "code",
54
  "metadata": {},
55
+ "outputs": [],
56
+ "execution_count": null,
57
  "source": [
58
  "#@title 1. 環境セットアップ\n",
59
+ "import os, sys, subprocess, json, math, time, gc, random\n",
 
60
  "from pathlib import Path\n",
61
  "\n",
62
+ "def _ensure(pkgs):\n",
63
+ " subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", *pkgs])\n",
 
64
  "\n",
65
+ "_ensure([\"transformers>=4.44\", \"accelerate\", \"datasets\", \"sentencepiece\", \"safetensors\", \"huggingface_hub\", \"numpy\"])\n",
66
  "\n",
 
67
  "import numpy as np\n",
68
+ "import torch\n",
 
 
69
  "\n",
70
+ "try:\n",
71
+ " from google.colab import userdata\n",
72
+ "except Exception:\n",
73
+ " userdata = None\n",
74
+ "\n",
75
+ "os.environ.setdefault(\"PYTORCH_CUDA_ALLOC_CONF\", \"expandable_segments:True\")\n",
76
+ "\n",
77
+ "def _secret(name):\n",
78
+ " v = os.environ.get(name)\n",
79
+ " if v:\n",
80
+ " return v.strip()\n",
81
+ " if userdata is not None:\n",
82
+ " try:\n",
83
+ " return (userdata.get(name) or \"\").strip()\n",
84
+ " except Exception:\n",
85
+ " pass\n",
86
+ " return \"\"\n",
87
+ "\n",
88
+ "HF_TOKEN = _secret(\"HF_TOKEN\") or _secret(\"HUGGINGFACE_TOKEN\")\n",
89
+ "DISCORD_WEBHOOK_URL = _secret(\"DISCORD_WEBHOOK_URL\")\n",
90
+ "if HF_TOKEN:\n",
91
+ " os.environ[\"HF_TOKEN\"] = HF_TOKEN\n",
92
+ " os.environ[\"HUGGING_FACE_HUB_TOKEN\"] = HF_TOKEN\n",
93
  "if DISCORD_WEBHOOK_URL:\n",
94
+ " os.environ[\"DISCORD_WEBHOOK_URL\"] = DISCORD_WEBHOOK_URL\n",
 
95
  "\n",
96
  "from huggingface_hub import login\n",
97
+ "if HF_TOKEN:\n",
98
+ " login(token=HF_TOKEN, add_to_git_credential=False)\n",
99
+ "\n",
100
+ "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
101
+ "print(\"device:\", device)\n",
102
+ "if device.type == \"cuda\":\n",
103
+ " print(torch.cuda.get_device_name(0))\n",
104
+ " print(\"VRAM GB:\", round(torch.cuda.get_device_properties(0).total_memory / 1024**3, 1))\n",
105
+ "print(\"HF_TOKEN:\", bool(HF_TOKEN), \"DISCORD:\", bool(DISCORD_WEBHOOK_URL))\n"
106
+ ]
107
  },
108
  {
109
  "cell_type": "code",
110
  "metadata": {},
111
+ "outputs": [],
112
+ "execution_count": null,
113
  "source": [
114
  "#@title 2. 設定\n",
 
115
  "REPO_ID = \"Mouserat/rat-77m\"\n",
116
  "DATASET_ID = \"Mouserat/llmjp4-2b-dataset\"\n",
117
  "BUCKET_ID = \"Mouserat/rat-77m\"\n",
 
119
  "SEQ_LEN = 1024\n",
120
  "VOCAB_SIZE = 64102\n",
121
  "MT5_VOCAB = 250112\n",
122
+ "TARGET_ROW_L2 = 0.5\n",
123
  "NOISE_DENSITY = 0.15\n",
124
  "MEAN_SPAN_LENGTH = 3.0\n",
125
  "SENTINEL_START = 64002\n",
126
+ "NUM_SENTINELS = 100\n",
127
  "\n",
128
+ "MICRO_BATCH = 32\n",
129
+ "GRAD_ACCUM = 4\n",
130
  "USE_BF16 = True\n",
131
  "USE_TF32 = True\n",
132
  "NUM_WORKERS = 4\n",
 
134
  "\n",
135
  "TOKEN_BUDGET = 2_000_000_000\n",
136
  "TOKENS_PER_STEP = MICRO_BATCH * GRAD_ACCUM * SEQ_LEN\n",
137
+ "MAX_STEPS = int(TOKEN_BUDGET // TOKENS_PER_STEP)\n",
138
  "WARMUP_STEPS = max(100, MAX_STEPS // 17)\n",
139
  "\n",
140
+ "LR = 1e-4\n",
141
  "WEIGHT_DECAY = 0.0\n",
142
+ "LOG_EVERY = 50\n",
143
  "DISCORD_EVERY = 500\n",
144
  "SAVE_EVERY = 1000\n",
145
  "BUCKET_REPLACE_EVERY = 5000\n",
146
  "EVAL_EVERY = 1000\n",
147
  "KEEP_LAST_N = 20\n",
148
  "GRAD_NORM_ABORT = 50.0\n",
149
+ "GRAD_NORM_STREAK = 2\n",
150
+ "UNK_RATE_ABORT = 0.02\n",
151
  "USE_ADAFACTOR = True\n",
152
  "LR_SCHEDULER = \"constant_with_warmup\"\n",
153
  "WALL_CLOCK_HOURS = 20.0\n",
154
  "\n",
155
+ "DIAG_LOSS_MIN, DIAG_LOSS_MAX = 8.0, 16.0\n",
156
+ "DIAG_GN_MAX = 50.0\n",
157
+ "DIAG_LOGITS_STD_MAX = 5.0\n",
158
+ "\n",
159
  "WORK_DIR = Path(\"/content/rat77m\") if Path(\"/content\").exists() else Path(\"./rat77m_work\")\n",
160
  "WORK_DIR.mkdir(parents=True, exist_ok=True)\n",
161
  "CKPT_DIR = WORK_DIR / \"checkpoints\"\n",
 
167
  " torch.backends.cuda.matmul.allow_tf32 = True\n",
168
  " torch.backends.cudnn.allow_tf32 = True\n",
169
  "\n",
170
+ "print(f\"MAX_STEPS={MAX_STEPS:,} (~{MAX_STEPS*TOKENS_PER_STEP/1e9:.2f}B) warmup={WARMUP_STEPS} lr={LR}\")\n",
171
+ "print(f\"batch={MICRO_BATCH}x{GRAD_ACCUM} tokens/step={TOKENS_PER_STEP:,} row_L2={TARGET_ROW_L2}\")\n"
172
+ ]
 
173
  },
174
  {
175
  "cell_type": "code",
176
  "metadata": {},
177
+ "outputs": [],
178
+ "execution_count": null,
179
  "source": [
180
  "#@title 3. Discord / Bucket ユーティリティ\n",
 
181
  "import urllib.request\n",
182
  "\n",
183
  "def discord_notify(content: str):\n",
184
+ " url = (os.environ.get(\"DISCORD_WEBHOOK_URL\") or globals().get(\"DISCORD_WEBHOOK_URL\") or \"\").strip()\n",
 
185
  " if not url:\n",
186
+ " print(\"[discord] skip: no webhook\")\n",
 
 
 
187
  " return\n",
188
  " if url.startswith(\"<\") and url.endswith(\">\"):\n",
189
  " url = url[1:-1]\n",
190
+ " data = json.dumps({\"content\": content[:1900], \"allowed_mentions\": {\"parse\": []}}).encode()\n",
 
 
 
191
  " req = urllib.request.Request(\n",
192
+ " url, data=data,\n",
193
+ " headers={\"Content-Type\": \"application/json\", \"User-Agent\": \"rat77m-trainer/1.0\", \"Accept\": \"application/json\"},\n",
 
 
 
 
 
194
  " method=\"POST\",\n",
195
  " )\n",
196
  " try:\n",
197
  " with urllib.request.urlopen(req, timeout=15) as resp:\n",
198
  " resp.read()\n",
199
+ " print(\"[discord] OK\")\n",
200
  " except Exception as e:\n",
201
  " body = \"\"\n",
202
  " if hasattr(e, \"read\"):\n",
 
205
  " except Exception:\n",
206
  " pass\n",
207
  " print(\"[discord]\", type(e).__name__, e, body)\n",
208
+ "\n",
209
  "def ensure_bucket():\n",
210
  " try:\n",
211
  " from huggingface_hub import create_bucket\n",
 
225
  " try:\n",
226
  " from huggingface_hub import batch_bucket_files\n",
227
  " base = Path(local_dir)\n",
228
+ " add = [(str(f), f\"{remote_prefix.strip('/')}/{f.relative_to(base).as_posix()}\")\n",
229
+ " for f in base.rglob(\"*\") if f.is_file()]\n",
230
  " for i in range(0, len(add), 64):\n",
231
  " batch_bucket_files(BUCKET_ID, add=add[i:i+64])\n",
232
  " print(\"[bucket] python OK\", dest)\n",
233
  " return True\n",
234
  " except Exception as e:\n",
235
+ " print(\"[bucket] failed\", getattr(r, \"stderr\", None) or e)\n",
236
  " return False\n",
237
  "\n",
238
  "ensure_bucket()\n",
239
+ "discord_notify(f\"rat-77m setup · MAX_STEPS={MAX_STEPS:,} · lr={LR} · row_L2={TARGET_ROW_L2}\")\n"
240
+ ]
 
 
 
241
  },
242
  {
243
  "cell_type": "code",
244
  "metadata": {},
245
+ "outputs": [],
246
+ "execution_count": null,
247
  "source": [
248
+ "#@title 4. 頻度リスト → 語彙マップ(旧 mT5 ID → 新 64k ID)\n",
 
249
  "from huggingface_hub import hf_hub_download\n",
250
+ "from transformers import AutoTokenizer\n",
251
  "\n",
252
+ "mt5_tok = AutoTokenizer.from_pretrained(\"google/mt5-small\", use_fast=False)\n",
253
+ "unk_oid = mt5_tok.unk_token_id\n",
254
+ "print(\"mT5 vocab_size\", mt5_tok.vocab_size, \"unk\", unk_oid)\n",
 
 
 
 
 
 
 
 
255
  "\n",
256
+ "freq_path = hf_hub_download(REPO_ID, \"vocab/token_frequency_top64k.jsonl\", token=HF_TOKEN or True)\n",
257
+ "id_remap = np.full(MT5_VOCAB, 2, dtype=np.int32)\n",
258
+ "id_remap[0] = 0\n",
259
+ "id_remap[1] = 1\n",
260
+ "if unk_oid is not None and 0 <= unk_oid < MT5_VOCAB:\n",
261
+ " id_remap[unk_oid] = 2\n",
262
+ "\n",
263
+ "normal_start = 3\n",
264
+ "count_normal = 0\n",
265
+ "with open(freq_path, \"r\", encoding=\"utf-8\") as f:\n",
266
  " for line in f:\n",
267
+ " if count_normal >= (SENTINEL_START - normal_start):\n",
 
 
 
 
268
  " break\n",
269
+ " obj = json.loads(line)\n",
270
+ " oid = obj.get(\"token_id\", obj.get(\"id\", obj.get(\"tid\")))\n",
271
+ " if oid is None:\n",
272
+ " continue\n",
273
+ " oid = int(oid)\n",
274
+ " if oid in (0, 1) or oid == unk_oid:\n",
275
+ " continue\n",
276
+ " if 0 <= oid < MT5_VOCAB:\n",
277
+ " id_remap[oid] = normal_start + count_normal\n",
278
+ " count_normal += 1\n",
279
+ "print(\"normal tokens mapped\", count_normal)\n",
 
 
 
 
 
 
 
 
 
 
 
280
  "\n",
281
  "def resolve_mt5_sentinel_ids(tokenizer, n=100):\n",
 
282
  " unk = tokenizer.unk_token_id\n",
 
283
  " if hasattr(tokenizer, \"get_sentinel_token_ids\"):\n",
284
  " try:\n",
285
  " ids = list(tokenizer.get_sentinel_token_ids())\n",
286
+ " if len(ids) >= n and all(i != unk for i in ids[:n]):\n",
287
+ " print(\"sentinel: get_sentinel_token_ids\")\n",
288
  " return ids[:n]\n",
289
  " except Exception as e:\n",
290
+ " print(\"get_sentinel_token_ids:\", e)\n",
 
291
  " ids = [tokenizer.convert_tokens_to_ids(f\"<extra_id_{i}>\") for i in range(n)]\n",
292
+ " if all(i != unk for i in ids):\n",
293
+ " print(\"sentinel: convert_tokens_to_ids\")\n",
294
  " return ids\n",
 
 
295
  " vs = int(tokenizer.vocab_size)\n",
296
  " ids = [vs - 1 - i for i in range(n)]\n",
297
+ " print(f\"sentinel: vocab tail fallback vocab_size={vs} extra_id_0={ids[0]}\")\n",
 
298
  " return ids\n",
299
  "\n",
300
+ "sentinel_old = resolve_mt5_sentinel_ids(mt5_tok, NUM_SENTINELS)\n",
301
+ "for i, oid in enumerate(sentinel_old):\n",
302
+ " assert isinstance(oid, int) and oid != unk_oid and 0 <= oid < MT5_VOCAB, (i, oid)\n",
 
 
 
 
 
 
 
 
303
  " id_remap[oid] = SENTINEL_START + i\n",
304
+ "print(\"sentinel new\", SENTINEL_START, \"..\", SENTINEL_START + NUM_SENTINELS - 1)\n"
305
+ ]
 
 
 
 
 
 
306
  },
307
  {
308
  "cell_type": "code",
309
  "metadata": {},
310
+ "outputs": [],
311
+ "execution_count": null,
312
  "source": [
313
+ "#@title 5. mT5 emb 移植(行 L2 = TARGET_ROW_L2)\n",
 
314
  "from transformers import AutoModelForSeq2SeqLM\n",
315
  "\n",
 
316
  "mt5 = AutoModelForSeq2SeqLM.from_pretrained(\"google/mt5-small\", torch_dtype=torch.float32)\n",
317
+ "mt5_emb = mt5.get_input_embeddings().weight.detach().float().cpu()\n",
318
+ "print(\"mt5 emb\", tuple(mt5_emb.shape), \"rowL2 mean\", mt5_emb.norm(dim=1).mean().item())\n",
319
+ "\n",
320
+ "new_emb = torch.zeros(VOCAB_SIZE, mt5_emb.size(1), dtype=torch.float32)\n",
321
+ "old_for_new = {}\n",
322
+ "for oid, nid in enumerate(id_remap.tolist()):\n",
323
+ " if 2 < nid < SENTINEL_START and nid not in old_for_new:\n",
324
+ " old_for_new[nid] = oid\n",
325
+ "for oid, nid in [(0, 0), (1, 1)]:\n",
326
+ " old_for_new[nid] = oid\n",
327
+ "if unk_oid is not None:\n",
328
+ " old_for_new[2] = int(unk_oid)\n",
329
+ "for i, oid in enumerate(sentinel_old):\n",
330
+ " old_for_new[SENTINEL_START + i] = int(oid)\n",
331
+ "\n",
332
+ "copied = 0\n",
333
+ "with torch.no_grad():\n",
334
+ " for nid, oid in old_for_new.items():\n",
335
+ " if 0 <= oid < mt5_emb.size(0) and 0 <= nid < VOCAB_SIZE:\n",
336
+ " new_emb[nid].copy_(mt5_emb[oid])\n",
337
+ " copied += 1\n",
338
+ " fill = (new_emb.abs().sum(dim=1) == 0)\n",
339
+ " nfill = int(fill.sum())\n",
340
+ " if nfill:\n",
341
+ " new_emb[fill] = torch.randn(nfill, new_emb.size(1)) * 0.02\n",
342
+ " row = new_emb.norm(dim=1, keepdim=True).clamp(min=1e-8)\n",
343
+ " new_emb.mul_(TARGET_ROW_L2 / row)\n",
344
+ "\n",
345
+ "print(\"copied\", copied, \"filled_rand\", nfill)\n",
346
+ "print(\"new rowL2 mean\", new_emb.norm(dim=1).mean().item())\n",
347
+ "print(\"new elem RMS mean\", new_emb.pow(2).mean(dim=1).sqrt().mean().item())\n",
348
  "del mt5, mt5_emb\n",
349
+ "gc.collect()\n",
350
+ "torch.cuda.empty_cache()\n"
351
+ ]
 
 
 
 
352
  },
353
  {
354
  "cell_type": "code",
355
  "metadata": {},
356
+ "outputs": [],
357
+ "execution_count": null,
358
  "source": [
359
+ "#@title 6. モデル初期化\n",
360
+ "from transformers import T5ForConditionalGeneration, T5Config\n",
 
 
 
361
  "\n",
362
  "config = T5Config(\n",
363
+ " vocab_size=VOCAB_SIZE,\n",
364
+ " d_model=512,\n",
365
+ " d_kv=64,\n",
366
+ " d_ff=2048,\n",
367
+ " num_layers=6,\n",
368
+ " num_decoder_layers=6,\n",
369
+ " num_heads=8,\n",
370
+ " relative_attention_num_buckets=32,\n",
371
+ " relative_attention_max_distance=128,\n",
372
+ " dropout_rate=0.1,\n",
373
+ " layer_norm_epsilon=1e-6,\n",
374
+ " initializer_factor=1.0,\n",
375
+ " feed_forward_proj=\"relu\",\n",
376
+ " is_encoder_decoder=True,\n",
377
+ " use_cache=True,\n",
378
+ " pad_token_id=0,\n",
379
+ " eos_token_id=1,\n",
380
+ " decoder_start_token_id=0,\n",
381
+ " tie_word_embeddings=True,\n",
382
  ")\n",
383
  "model = T5ForConditionalGeneration(config)\n",
 
384
  "with torch.no_grad():\n",
385
  " model.get_input_embeddings().weight.copy_(new_emb)\n",
386
+ " out_emb = model.get_output_embeddings()\n",
387
+ " if out_emb is not None and out_emb.weight.data_ptr() != model.get_input_embeddings().weight.data_ptr():\n",
388
+ " out_emb.weight.copy_(new_emb)\n",
389
  "\n",
390
+ "model.to(device)\n",
391
+ "print(f\"params={model.num_parameters():,}\")\n",
392
+ "print(\"emb rowL2\", model.get_input_embeddings().weight.float().norm(dim=1).mean().item())\n"
393
+ ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
394
  },
395
  {
396
  "cell_type": "code",
397
  "metadata": {},
398
+ "outputs": [],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
399
  "execution_count": null,
400
+ "source": [
401
+ "#@title 7. init-fixed 保存(任意)\n",
402
+ "save_init = WORK_DIR / \"init-fixed\"\n",
403
+ "save_init.mkdir(exist_ok=True)\n",
404
+ "model.cpu()\n",
405
+ "model.save_pretrained(save_init)\n",
406
+ "config.save_pretrained(save_init)\n",
407
+ "print(\"saved\", save_init)\n",
408
+ "model.to(device)\n"
409
+ ]
410
  },
411
  {
412
  "cell_type": "code",
413
  "metadata": {},
414
+ "outputs": [],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
415
  "execution_count": null,
416
+ "source": [
417
+ "#@title 8. データセット取得 + 旧 ID 確認\n",
418
+ "from huggingface_hub import snapshot_download, list_repo_files\n",
419
+ "\n",
420
+ "files = list_repo_files(DATASET_ID, repo_type=\"dataset\", token=HF_TOKEN or True)\n",
421
+ "bin_files = [f for f in files if f.endswith(\".bin\")]\n",
422
+ "print(\"bin files\", len(bin_files), bin_files[:8])\n",
423
+ "\n",
424
+ "data_root = snapshot_download(DATASET_ID, repo_type=\"dataset\", local_dir=str(DATA_DIR), token=HF_TOKEN or True)\n",
425
+ "data_root = Path(data_root)\n",
426
+ "bins = sorted(data_root.rglob(\"*.bin\"))\n",
427
+ "print(\"local bins\", len(bins))\n",
428
+ "assert bins, \"no .bin found\"\n",
429
+ "\n",
430
+ "sample = np.fromfile(bins[0], dtype=\"<u4\", count=min(1024 * 8, os.path.getsize(bins[0]) // 4))\n",
431
+ "n = (len(sample) // SEQ_LEN) * SEQ_LEN\n",
432
+ "sample = sample[:n].reshape(-1, SEQ_LEN)\n",
433
+ "flat = sample.reshape(-1)\n",
434
+ "vals, cnts = np.unique(flat, return_counts=True)\n",
435
+ "top = vals[cnts.argmax()]\n",
436
+ "print(\"most common OLD id\", int(top), \"freq\", float(cnts.max()) / len(flat))\n"
437
+ ]
438
  },
439
  {
440
  "cell_type": "code",
441
  "metadata": {},
442
+ "outputs": [],
443
+ "execution_count": null,
444
  "source": [
445
+ "#@title 9. Dataset(再マップ)+ Span Collator + train/eval 分割\n",
 
446
  "from torch.utils.data import Dataset, Subset\n",
447
  "\n",
448
+ "class BinTokDataset(Dataset):\n",
449
+ " def __init__(self, paths, seq_len, remap):\n",
 
450
  " self.seq_len = seq_len\n",
451
+ " self.remap = remap\n",
452
+ " self.paths = [Path(p) for p in paths]\n",
453
+ " self.index = []\n",
454
+ " for pi, p in enumerate(self.paths):\n",
455
+ " ntok = p.stat().st_size // 4\n",
456
+ " nseq = ntok // seq_len\n",
457
+ " for s in range(nseq):\n",
458
+ " self.index.append((pi, s * seq_len))\n",
459
+ " print(f\"dataset sequences={len(self.index):,} files={len(self.paths)}\")\n",
 
 
460
  "\n",
461
  " def __len__(self):\n",
462
+ " return len(self.index)\n",
463
+ "\n",
464
+ " def __getitem__(self, i):\n",
465
+ " pi, off = self.index[i]\n",
466
+ " raw = np.fromfile(self.paths[pi], dtype=\"<u4\", count=self.seq_len, offset=off * 4)\n",
467
+ " raw = np.clip(raw, 0, len(self.remap) - 1).astype(np.int64)\n",
468
+ " ids = self.remap[raw]\n",
469
+ " ids = np.where((ids >= 0) & (ids < VOCAB_SIZE), ids, 2).astype(np.int64)\n",
470
+ " return {\"input_ids\": torch.from_numpy(ids)}\n",
471
+ "\n",
472
+ "all_ds = BinTokDataset(bins, SEQ_LEN, id_remap)\n",
473
+ "\n",
474
+ "rng = np.random.default_rng(42)\n",
475
+ "idxs = rng.choice(len(all_ds), size=min(2000, len(all_ds)), replace=False)\n",
476
+ "unk_n = tot = 0\n",
477
+ "for i in idxs:\n",
478
+ " x = all_ds[int(i)][\"input_ids\"]\n",
479
+ " unk_n += int((x == 2).sum())\n",
480
+ " tot += x.numel()\n",
481
+ "unk_rate = unk_n / tot\n",
482
+ "print(f\"unk_rate={unk_rate:.4%} on {len(idxs)} seqs\")\n",
483
+ "if unk_rate > UNK_RATE_ABORT:\n",
484
+ " raise RuntimeError(f\"unk_rate {unk_rate:.4%} > {UNK_RATE_ABORT:.2%}\")\n",
485
+ "\n",
486
+ "perm = rng.permutation(len(all_ds))\n",
487
+ "n_eval = max(256, int(len(all_ds) * 0.015))\n",
488
+ "train_ds = Subset(all_ds, perm[n_eval:].tolist())\n",
489
+ "eval_ds = Subset(all_ds, perm[:n_eval].tolist())\n",
490
+ "print(\"train\", len(train_ds), \"eval\", len(eval_ds))\n",
491
+ "\n",
492
+ "def random_spans_noise_mask(length, noise_density, mean_span_length, rng):\n",
493
+ " num_noise = int(round(length * noise_density))\n",
494
+ " num_noise = max(1, min(length - 1, num_noise))\n",
495
+ " num_spans = max(1, int(round(num_noise / mean_span_length)))\n",
496
+ " span_starts = sorted(rng.choice(length, size=min(num_spans, length), replace=False).tolist())\n",
497
+ " mask = np.zeros(length, dtype=bool)\n",
498
+ " remaining = num_noise\n",
499
+ " for s in span_starts:\n",
500
+ " if remaining <= 0:\n",
501
+ " break\n",
502
+ " span_len = min(max(1, int(round(mean_span_length))), remaining, length - s)\n",
503
+ " mask[s:s + span_len] = True\n",
504
+ " remaining -= span_len\n",
505
  " i = 0\n",
506
+ " while remaining > 0 and i < length:\n",
507
+ " if not mask[i]:\n",
508
+ " mask[i] = True\n",
509
+ " remaining -= 1\n",
510
+ " i += 1\n",
511
+ " return mask\n",
512
+ "\n",
513
+ "def span_corrupt(input_ids, noise_density, mean_span_length, sentinel_start, seed=None):\n",
514
+ " rng = np.random.default_rng(seed)\n",
515
+ " ids = input_ids.numpy() if torch.is_tensor(input_ids) else np.asarray(input_ids)\n",
516
+ " L = len(ids)\n",
517
+ " mask = random_spans_noise_mask(L, noise_density, mean_span_length, rng)\n",
518
+ " enc, labels, sent_i, i = [], [], 0, 0\n",
519
+ " while i < L:\n",
520
+ " if not mask[i]:\n",
521
+ " enc.append(int(ids[i]))\n",
522
  " i += 1\n",
523
+ " else:\n",
524
+ " enc.append(sentinel_start + sent_i)\n",
525
+ " labels.append(sentinel_start + sent_i)\n",
526
+ " while i < L and mask[i]:\n",
527
+ " labels.append(int(ids[i]))\n",
528
+ " i += 1\n",
529
+ " sent_i = min(sent_i + 1, NUM_SENTINELS - 1)\n",
530
+ " labels.append(1)\n",
531
+ " return enc, labels\n",
532
  "\n",
533
  "class SpanDataCollator:\n",
 
 
 
 
 
 
534
  " def __call__(self, features):\n",
535
+ " enc_batch, lab_batch = [], []\n",
 
536
  " for f in features:\n",
537
+ " enc, lab = span_corrupt(f[\"input_ids\"], NOISE_DENSITY, MEAN_SPAN_LENGTH, SENTINEL_START, seed=None)\n",
538
+ " enc_batch.append(enc[:SEQ_LEN])\n",
539
+ " lab_batch.append(lab[:SEQ_LEN])\n",
540
+ " max_e = max(len(x) for x in enc_batch)\n",
541
+ " max_l = max(len(x) for x in lab_batch)\n",
542
+ " input_ids, attention_mask, labels = [], [], []\n",
543
+ " for enc, lab in zip(enc_batch, lab_batch):\n",
544
+ " pe, pl = max_e - len(enc), max_l - len(lab)\n",
545
+ " input_ids.append(enc + [0] * pe)\n",
546
+ " attention_mask.append([1] * len(enc) + [0] * pe)\n",
547
+ " labels.append(lab + [-100] * pl)\n",
548
+ " return {\n",
549
+ " \"input_ids\": torch.tensor(input_ids, dtype=torch.long),\n",
550
+ " \"attention_mask\": torch.tensor(attention_mask, dtype=torch.long),\n",
551
+ " \"labels\": torch.tensor(labels, dtype=torch.long),\n",
552
+ " }\n",
553
+ "\n",
554
+ "class EvalSpanCollator:\n",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
555
  " def __call__(self, features):\n",
556
+ " enc_batch, lab_batch = [], []\n",
557
+ " for j, f in enumerate(features):\n",
558
+ " base = int(f[\"input_ids\"][:8].sum().item()) if hasattr(f[\"input_ids\"], \"sum\") else 0\n",
559
+ " enc, lab = span_corrupt(f[\"input_ids\"], NOISE_DENSITY, MEAN_SPAN_LENGTH, SENTINEL_START, seed=base + j * 10007)\n",
560
+ " enc_batch.append(enc[:SEQ_LEN])\n",
561
+ " lab_batch.append(lab[:SEQ_LEN])\n",
562
+ " max_e = max(len(x) for x in enc_batch)\n",
563
+ " max_l = max(len(x) for x in lab_batch)\n",
564
+ " input_ids, attention_mask, labels = [], [], []\n",
565
+ " for enc, lab in zip(enc_batch, lab_batch):\n",
566
+ " pe, pl = max_e - len(enc), max_l - len(lab)\n",
567
+ " input_ids.append(enc + [0] * pe)\n",
568
+ " attention_mask.append([1] * len(enc) + [0] * pe)\n",
569
+ " labels.append(lab + [-100] * pl)\n",
570
  " return {\n",
571
+ " \"input_ids\": torch.tensor(input_ids, dtype=torch.long),\n",
572
+ " \"attention_mask\": torch.tensor(attention_mask, dtype=torch.long),\n",
573
+ " \"labels\": torch.tensor(labels, dtype=torch.long),\n",
574
  " }\n",
575
  "\n",
576
+ "print(\"collators ready\")\n"
577
+ ]
 
 
 
 
578
  },
579
  {
580
  "cell_type": "code",
581
  "metadata": {},
582
+ "outputs": [],
583
+ "execution_count": null,
584
  "source": [
585
+ "#@title 10. 学習前診断(不合格なら raise)\n",
 
 
 
 
586
  "model.train()\n",
587
  "collator = SpanDataCollator()\n",
588
  "batch = collator([train_ds[i] for i in range(min(8, len(train_ds)))])\n",
589
  "batch = {k: v.to(device) for k, v in batch.items()}\n",
590
  "\n",
591
+ "valid = (batch[\"labels\"] != -100).sum().item()\n",
592
+ "print(\"valid label tokens\", valid, \"/\", batch[\"labels\"].numel())\n",
593
+ "labs = batch[\"labels\"][batch[\"labels\"] != -100]\n",
594
+ "print(\"label id range\", int(labs.min()), int(labs.max()))\n",
595
+ "\n",
596
  "model.zero_grad(set_to_none=True)\n",
597
+ "out = model(**batch)\n",
598
+ "loss = float(out.loss.detach().float().cpu())\n",
599
+ "logits_std = float(out.logits.detach().float().std().cpu())\n",
600
+ "out.loss.backward()\n",
601
+ "gn = float(torch.nn.utils.clip_grad_norm_(model.parameters(), 1e9).cpu())\n",
602
  "model.zero_grad(set_to_none=True)\n",
603
  "\n",
604
+ "print(f\"diag loss={loss:.4f} (expect {DIAG_LOSS_MIN}..{DIAG_LOSS_MAX})\")\n",
605
+ "print(f\"diag grad_norm={gn:.4f} (expect < {DIAG_GN_MAX})\")\n",
606
+ "print(f\"diag logits_std={logits_std:.4f} (expect < {DIAG_LOGITS_STD_MAX})\")\n",
607
+ "\n",
608
+ "ok = (DIAG_LOSS_MIN <= loss <= DIAG_LOSS_MAX) and (gn < DIAG_GN_MAX) and (logits_std < DIAG_LOGITS_STD_MAX)\n",
 
 
 
 
 
 
 
 
 
 
609
  "if not ok:\n",
610
+ " discord_notify(f\"❌ PRE-TRAIN DIAG FAIL loss={loss:.4f} gn={gn:.4f} logits_std={logits_std:.4f}\")\n",
611
+ " raise RuntimeError(\n",
612
+ " f\"PRE-TRAIN DIAG FAIL: loss={loss:.4f} grad_norm={gn:.4f} logits_std={logits_std:.4f}. 学習禁止。\"\n",
613
+ " )\n",
614
  "print(\"PRE-TRAIN DIAG OK\")\n",
615
+ "discord_notify(f\"✅ PRE-TRAIN DIAG OK loss={loss:.4f} gn={gn:.4f} logits_std={logits_std:.4f}\")\n"
616
+ ]
 
 
 
617
  },
618
  {
619
  "cell_type": "code",
620
  "metadata": {},
621
+ "outputs": [],
622
+ "execution_count": null,
623
  "source": [
624
  "#@title 11. Trainer + Discord + Bucket + 連続 grad 停止 + 再開\n",
625
+ "from transformers import Trainer, TrainingArguments, TrainerCallback, Adafactor\n",
626
+ "import inspect\n",
 
627
  "\n",
628
  "class EvalSwapTrainer(Trainer):\n",
 
629
  " def __init__(self, *args, eval_data_collator=None, **kwargs):\n",
630
  " super().__init__(*args, **kwargs)\n",
 
631
  " self._eval_collator = eval_data_collator or self.data_collator\n",
632
  "\n",
633
  " def get_eval_dataloader(self, eval_dataset=None):\n",
 
634
  " original = self.data_collator\n",
635
  " self.data_collator = self._eval_collator\n",
636
  " try:\n",
 
648
  " self.t0 = time.time()\n",
649
  " self.last = 0\n",
650
  " self.warmup_alerted = set()\n",
651
+ " self._loss_high_since = None\n",
652
+ "\n",
653
  " def on_log(self, args, state, control, logs=None, **kwargs):\n",
654
  " if not logs or state.global_step <= 0:\n",
655
  " return control\n",
656
  " step = state.global_step\n",
657
+ " loss, gn, lr = logs.get(\"loss\"), logs.get(\"grad_norm\"), logs.get(\"learning_rate\")\n",
658
+ " eval_loss = logs.get(\"eval_loss\")\n",
659
+ " if loss is not None:\n",
660
+ " print(f\"step {step}/{MAX_STEPS} loss={float(loss):.4f} grad_norm={float(gn) if gn is not None else 'NA'} lr={lr} eval_loss={eval_loss}\")\n",
661
  " if loss is not None and float(loss) > 35:\n",
 
 
662
  " if self._loss_high_since is None:\n",
663
  " self._loss_high_since = step\n",
664
+ " discord_notify(f\"⚠️ **loss high (first)** step `{step}` loss`{loss}` grad_norm`{gn}` lr`{lr}`\")\n",
 
 
665
  " elif step % 500 == 0:\n",
666
+ " discord_notify(f\"⚠️ **loss still high** step `{step}` loss`{loss}` grad_norm`{gn}` lr`{lr}`\")\n",
 
 
667
  " else:\n",
668
  " self._loss_high_since = None\n",
 
669
  " for mult, tag in [(0.8, \"warmup-80%\"), (1.0, \"warmup-END\"), (1.2, \"post-warmup+20%\")]:\n",
670
  " mark = int(self.warmup_steps * mult)\n",
671
  " if mark > 0 and step >= mark and tag not in self.warmup_alerted:\n",
672
  " self.warmup_alerted.add(tag)\n",
673
+ " discord_notify(f\"⚠️ **{tag}** step `{step}` loss`{loss}` lr`{lr}` grad_norm`{gn}`\")\n",
 
 
 
 
 
 
674
  " if step % self.every != 0 or step == self.last:\n",
675
  " return control\n",
676
  " self.last = step\n",
677
  " tok = step * TOKENS_PER_STEP\n",
678
+ " discord_notify(\n",
679
  " f\"**rat-77m** step `{step}/{MAX_STEPS}`\\n\"\n",
680
+ " f\"loss`{loss}` eval_loss`{eval_loss}` lr`{lr}` grad_norm`{gn}`\\n\"\n",
681
+ " f\"tokens`{tok/1e9:.3f}B` elapsed`{(time.time()-self.t0)/3600:.2f}h`\"\n",
 
682
  " )\n",
 
 
683
  " return control\n",
684
  "\n",
685
  "class BucketReplaceCallback(TrainerCallback):\n",
 
692
  " return control\n",
693
  " latest = ckpts[-1]\n",
694
  " bucket_sync_dir(str(latest), f\"{BUCKET_RUN}/{latest.name}\")\n",
695
+ " if state.global_step % self.every == 0 and state.global_step > 0:\n",
696
  " bucket_sync_dir(str(latest), BUCKET_LATEST)\n",
697
  " discord_notify(f\"Bucket latest @ {state.global_step}: {latest.name}\")\n",
698
  " return control\n",
699
  "\n",
700
  "class GradNormStreakCallback(TrainerCallback):\n",
701
+ " def __init__(self, thr=50.0, streak=2):\n",
702
+ " self.thr, self.streak_need, self.streak = thr, streak, 0\n",
 
 
703
  " def on_log(self, args, state, control, logs=None, **kwargs):\n",
704
  " if not logs or logs.get(\"grad_norm\") is None:\n",
705
  " return control\n",
 
707
  " if gn > self.thr:\n",
708
  " self.streak += 1\n",
709
  " print(f\"grad_norm spike {gn:.2f} streak={self.streak}/{self.streak_need}\")\n",
 
710
  " if self.streak == 1 or self.streak >= self.streak_need:\n",
711
+ " discord_notify(f\"⚠️ **grad_norm spike** `{gn:.2f}` streak={self.streak}/{self.streak_need} step `{state.global_step}`\")\n",
 
 
712
  " else:\n",
713
  " self.streak = 0\n",
714
  " if self.streak >= self.streak_need:\n",
 
720
  "\n",
721
  "class WallClockStopCallback(TrainerCallback):\n",
722
  " def __init__(self, hours=20.0):\n",
723
+ " self.limit, self.t0 = hours * 3600, time.time()\n",
 
724
  " def on_step_end(self, args, state, control, **kwargs):\n",
725
  " if time.time() - self.t0 >= self.limit:\n",
 
726
  " discord_notify(f\"Wall clock stop step={state.global_step}\")\n",
727
  " control.should_training_stop = True\n",
728
  " control.should_save = True\n",
729
  " return control\n",
730
  "\n",
731
  "def try_resume_from_bucket():\n",
 
732
  " local = CKPT_DIR / \"resume\"\n",
733
  " local.mkdir(exist_ok=True)\n",
734
  " candidates = []\n",
735
  " for prefix in [BUCKET_LATEST, BUCKET_RUN]:\n",
736
  " dest = Path(local) / Path(prefix).name.replace(\"/\", \"_\")\n",
737
+ " r = subprocess.run([\"hf\", \"buckets\", \"sync\", f\"hf://buckets/{BUCKET_ID}/{prefix}\", str(dest)], capture_output=True, text=True)\n",
 
 
 
738
  " if r.returncode != 0:\n",
739
  " continue\n",
740
  " for p in dest.rglob(\"checkpoint-*\"):\n",
741
  " if p.is_dir() and p.name.split(\"-\")[-1].isdigit():\n",
742
  " candidates.append((int(p.name.split(\"-\")[-1]), p))\n",
743
  " if (dest / \"config.json\").exists():\n",
 
744
  " candidates.append((0, dest))\n",
745
  " for p in CKPT_DIR.glob(\"checkpoint-*\"):\n",
746
  " if p.is_dir() and p.name.split(\"-\")[-1].isdigit():\n",
 
755
  "resume_path = try_resume_from_bucket()\n",
756
  "print(\"resume_path:\", resume_path)\n",
757
  "\n",
 
758
  "_ta_kwargs = dict(\n",
759
+ " output_dir=str(CKPT_DIR), max_steps=MAX_STEPS,\n",
760
+ " per_device_train_batch_size=MICRO_BATCH, gradient_accumulation_steps=GRAD_ACCUM,\n",
761
+ " learning_rate=LR, weight_decay=WEIGHT_DECAY, warmup_steps=WARMUP_STEPS,\n",
762
+ " lr_scheduler_type=LR_SCHEDULER, logging_strategy=\"steps\", logging_steps=LOG_EVERY,\n",
763
+ " eval_steps=EVAL_EVERY, save_strategy=\"steps\", save_steps=SAVE_EVERY, save_total_limit=KEEP_LAST_N,\n",
764
+ " bf16=USE_BF16 and device.type == \"cuda\", dataloader_num_workers=NUM_WORKERS,\n",
765
+ " dataloader_pin_memory=PIN_MEMORY, remove_unused_columns=False, report_to=\"none\",\n",
766
+ " max_grad_norm=1.0, disable_tqdm=False,\n",
 
 
 
 
 
 
 
 
 
 
 
 
767
  ")\n",
768
  "_sig = inspect.signature(TrainingArguments.__init__)\n",
769
  "if \"eval_strategy\" in _sig.parameters:\n",
 
776
  " _ta_kwargs[\"tf32\"] = USE_TF32\n",
777
  "if \"optim\" in _sig.parameters:\n",
778
  " _ta_kwargs[\"optim\"] = \"adafactor\" if USE_ADAFACTOR else \"adamw_torch\"\n",
779
+ "if \"logging_first_step\" in _sig.parameters:\n",
780
+ " _ta_kwargs[\"logging_first_step\"] = True\n",
781
+ "\n",
782
  "training_args = TrainingArguments(**_ta_kwargs)\n",
783
+ "print(\"TrainingArguments:\", sorted(_ta_kwargs.keys()))\n",
784
  "\n",
785
  "optimizer = None\n",
786
  "if USE_ADAFACTOR:\n",
 
791
  " )\n",
792
  "\n",
793
  "trainer_kw = dict(\n",
794
+ " model=model, args=training_args, train_dataset=train_ds, eval_dataset=eval_ds,\n",
 
 
 
795
  " data_collator=SpanDataCollator(),\n",
796
  " callbacks=[\n",
797
  " DiscordCallback(DISCORD_EVERY, warmup_steps=WARMUP_STEPS),\n",
 
802
  ")\n",
803
  "if optimizer is not None:\n",
804
  " trainer_kw[\"optimizers\"] = (optimizer, None)\n",
 
805
  "trainer_kw[\"eval_data_collator\"] = EvalSpanCollator()\n",
806
  "trainer = EvalSwapTrainer(**trainer_kw)\n",
807
+ "\n",
808
+ "discord_notify(f\"**rat-77m TRAIN START** steps={MAX_STEPS} lr={LR} resume={resume_path}\")\n",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
809
  "train_result = trainer.train(resume_from_checkpoint=resume_path)\n",
810
  "print(train_result)\n",
811
  "discord_notify(f\"Training finished {train_result.metrics}\")\n"
812
+ ]
 
 
813
  },
814
  {
815
  "cell_type": "code",
816
  "metadata": {},
817
+ "outputs": [],
818
+ "execution_count": null,
819
  "source": [
820
+ "#@title 12. 完了処理: final-20 を Bucket へ\n",
 
 
 
821
  "ckpts = sorted([p for p in CKPT_DIR.glob(\"checkpoint-*\") if p.is_dir()],\n",
822
+ " key=lambda p: int(p.name.split(\"-\")[-1]))\n",
823
+ "final20 = ckpts[-20:]\n",
824
+ "print(\"final20\", [p.name for p in final20])\n",
825
+ "for p in final20:\n",
826
+ " bucket_sync_dir(str(p), f\"final-20/{p.name}\")\n",
827
  "\n",
828
  "final_dir = WORK_DIR / \"final\"\n",
829
  "final_dir.mkdir(exist_ok=True)\n",
830
  "trainer.save_model(str(final_dir))\n",
 
 
831
  "bucket_sync_dir(str(final_dir), \"final\")\n",
832
+ "try:\n",
833
+ " from huggingface_hub import HfApi\n",
834
+ " HfApi(token=HF_TOKEN or True).upload_folder(\n",
835
+ " folder_path=str(final_dir), repo_id=REPO_ID, repo_type=\"model\", path_in_repo=\"final\"\n",
836
+ " )\n",
837
+ " print(\"uploaded final/ to\", REPO_ID)\n",
838
+ "except Exception as e:\n",
839
+ " print(\"repo upload:\", e)\n",
840
+ "discord_notify(f\"✅ rat-77m complete · ckpts={len(ckpts)}\")\n",
841
+ "print(\"done\")\n"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
842
  ]
843
  }
844
  ]