Mouserat commited on
Commit
ffc5dfd
·
verified ·
1 Parent(s): db72e85

Upload notebooks/cells/cell02_2__設定.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. notebooks/cells/cell02_2__設定.py +54 -0
notebooks/cells/cell02_2__設定.py ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #@title 2. 設定
2
+
3
+ REPO_ID = "Mouserat/rat-77m"
4
+ DATASET_ID = "Mouserat/llmjp4-2b-dataset"
5
+ BUCKET_ID = "Mouserat/rat-77m"
6
+
7
+ SEQ_LEN = 1024
8
+ VOCAB_SIZE = 64102
9
+ MT5_VOCAB = 250112
10
+ TARGET_ROW_L2 = 0.5 # v6実績: 行 L2=0.5(要素RMS≈0.022)。要素RMS=0.5は学習で loss≈89 に発散
11
+ NOISE_DENSITY = 0.15
12
+ MEAN_SPAN_LENGTH = 3.0
13
+ SENTINEL_START = 64002
14
+ SENTINEL_END = 64101
15
+
16
+ MICRO_BATCH = 64
17
+ GRAD_ACCUM = 2
18
+ USE_BF16 = True
19
+ USE_TF32 = True
20
+ NUM_WORKERS = 4
21
+ PIN_MEMORY = True
22
+
23
+ TOKEN_BUDGET = 2_000_000_000
24
+ TOKENS_PER_STEP = MICRO_BATCH * GRAD_ACCUM * SEQ_LEN
25
+ MAX_STEPS = TOKEN_BUDGET // TOKENS_PER_STEP
26
+ WARMUP_STEPS = max(100, MAX_STEPS // 17)
27
+
28
+ LR = 1e-4 # 仕様寄り・前回 3e-4 で危険だったため抑制
29
+ WEIGHT_DECAY = 0.0
30
+ LOG_EVERY = 50 # grad_norm 監視を細かく
31
+ DISCORD_EVERY = 500
32
+ SAVE_EVERY = 1000
33
+ BUCKET_REPLACE_EVERY = 5000
34
+ EVAL_EVERY = 1000
35
+ KEEP_LAST_N = 20
36
+ GRAD_NORM_ABORT = 50.0
37
+ GRAD_NORM_STREAK = 2 # 旧ドキュメント運用: 連続2回
38
+ UNK_RATE_ABORT = 0.02 # 再マップ後 unk 率の上限(期待 ~0.5%)
39
+ USE_ADAFACTOR = True
40
+ LR_SCHEDULER = "constant_with_warmup"
41
+ WALL_CLOCK_HOURS = 20.0
42
+
43
+ WORK_DIR = Path("/content/rat77m") if Path("/content").exists() else Path("./rat77m_work")
44
+ WORK_DIR.mkdir(parents=True, exist_ok=True)
45
+ CKPT_DIR = WORK_DIR / "checkpoints"
46
+ CKPT_DIR.mkdir(exist_ok=True)
47
+ DATA_DIR = WORK_DIR / "data"
48
+ DATA_DIR.mkdir(exist_ok=True)
49
+
50
+ if USE_TF32 and torch.cuda.is_available():
51
+ torch.backends.cuda.matmul.allow_tf32 = True
52
+ torch.backends.cudnn.allow_tf32 = True
53
+
54
+ print(f"MAX_STEPS={MAX_STEPS:,} (~{MAX_STEPS*TOKENS_PER_STEP/1e9:.2f}B) warmup={WARMUP_STEPS} lr={LR} grad_abort={GRAD_NORM_STREAK}x>{GRAD_NORM_ABORT}")