Upload notebooks/cells/cell02_2__設定.py with huggingface_hub
Browse files
notebooks/cells/cell02_2__設定.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#@title 2. 設定
|
| 2 |
+
|
| 3 |
+
REPO_ID = "Mouserat/rat-77m"
|
| 4 |
+
DATASET_ID = "Mouserat/llmjp4-2b-dataset"
|
| 5 |
+
BUCKET_ID = "Mouserat/rat-77m"
|
| 6 |
+
|
| 7 |
+
SEQ_LEN = 1024
|
| 8 |
+
VOCAB_SIZE = 64102
|
| 9 |
+
MT5_VOCAB = 250112
|
| 10 |
+
TARGET_ROW_L2 = 0.5 # v6実績: 行 L2=0.5(要素RMS≈0.022)。要素RMS=0.5は学習で loss≈89 に発散
|
| 11 |
+
NOISE_DENSITY = 0.15
|
| 12 |
+
MEAN_SPAN_LENGTH = 3.0
|
| 13 |
+
SENTINEL_START = 64002
|
| 14 |
+
SENTINEL_END = 64101
|
| 15 |
+
|
| 16 |
+
MICRO_BATCH = 64
|
| 17 |
+
GRAD_ACCUM = 2
|
| 18 |
+
USE_BF16 = True
|
| 19 |
+
USE_TF32 = True
|
| 20 |
+
NUM_WORKERS = 4
|
| 21 |
+
PIN_MEMORY = True
|
| 22 |
+
|
| 23 |
+
TOKEN_BUDGET = 2_000_000_000
|
| 24 |
+
TOKENS_PER_STEP = MICRO_BATCH * GRAD_ACCUM * SEQ_LEN
|
| 25 |
+
MAX_STEPS = TOKEN_BUDGET // TOKENS_PER_STEP
|
| 26 |
+
WARMUP_STEPS = max(100, MAX_STEPS // 17)
|
| 27 |
+
|
| 28 |
+
LR = 1e-4 # 仕様寄り・前回 3e-4 で危険だったため抑制
|
| 29 |
+
WEIGHT_DECAY = 0.0
|
| 30 |
+
LOG_EVERY = 50 # grad_norm 監視を細かく
|
| 31 |
+
DISCORD_EVERY = 500
|
| 32 |
+
SAVE_EVERY = 1000
|
| 33 |
+
BUCKET_REPLACE_EVERY = 5000
|
| 34 |
+
EVAL_EVERY = 1000
|
| 35 |
+
KEEP_LAST_N = 20
|
| 36 |
+
GRAD_NORM_ABORT = 50.0
|
| 37 |
+
GRAD_NORM_STREAK = 2 # 旧ドキュメント運用: 連続2回
|
| 38 |
+
UNK_RATE_ABORT = 0.02 # 再マップ後 unk 率の上限(期待 ~0.5%)
|
| 39 |
+
USE_ADAFACTOR = True
|
| 40 |
+
LR_SCHEDULER = "constant_with_warmup"
|
| 41 |
+
WALL_CLOCK_HOURS = 20.0
|
| 42 |
+
|
| 43 |
+
WORK_DIR = Path("/content/rat77m") if Path("/content").exists() else Path("./rat77m_work")
|
| 44 |
+
WORK_DIR.mkdir(parents=True, exist_ok=True)
|
| 45 |
+
CKPT_DIR = WORK_DIR / "checkpoints"
|
| 46 |
+
CKPT_DIR.mkdir(exist_ok=True)
|
| 47 |
+
DATA_DIR = WORK_DIR / "data"
|
| 48 |
+
DATA_DIR.mkdir(exist_ok=True)
|
| 49 |
+
|
| 50 |
+
if USE_TF32 and torch.cuda.is_available():
|
| 51 |
+
torch.backends.cuda.matmul.allow_tf32 = True
|
| 52 |
+
torch.backends.cudnn.allow_tf32 = True
|
| 53 |
+
|
| 54 |
+
print(f"MAX_STEPS={MAX_STEPS:,} (~{MAX_STEPS*TOKENS_PER_STEP/1e9:.2f}B) warmup={WARMUP_STEPS} lr={LR} grad_abort={GRAD_NORM_STREAK}x>{GRAD_NORM_ABORT}")
|