File size: 4,063 Bytes
7a936fb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 |
use_liger=False
xla_on_gpu=False
# ---------------------------------------------------------------------------
# Core shape -- chosen for GPU efficiency, not just "a number that fits"
# ---------------------------------------------------------------------------
CONTEXT = 1024
max_gen_tokens = 1024
gen_headroom = 128
vocab_size = 8192+4096
# ── Engram (static n-gram memory) ──────────────────────────────────
use_engram = True
engram_layer_ids = (1, 8) # two layers, one early, one mid
engram_max_ngram = 3 # 2-gram + 3-gram tables
engram_vocab_size = (4096, 4096) # halved from default
engram_embed_per_ngram = 128
engram_n_heads = 4
engram_kernel_size = 4
engram_seed = 0
engram_compressed_vocab = 0 # 0 → set from tokenizer lookup
bias = False
d_rope = None
top_k = 64
moe_top_k = 2
use_triton_fusions = False
gradient_checkpointing = False
num_kv_heads = 4
pattern = "dense"
sliding_window_size = 512
tensor_parallel_size = 1
# ─────────────────────────────────────────────────────────────────────
# Curriculum learning (token-level mixing across sources)
# ─────────────────────────────────────────────────────────────────────
# Master switch. False = old behavior (uniform random over the whole
# flat token array). True = per-source anchor schedule below.
CURRICULUM_ENABLED = False
# Number of training steps the anchor schedule spans. Set equal to
# MAX_STEPS in train.py so the ramp completes at run end.
CURRICULUM_TOTAL_STEPS = 20000
# Sources, in curriculum order (easiest first). Each entry:
# (name, [glob_patterns], w@0, w@T/3, w@2T/3, w@T)
# Between anchors, weights interpolate linearly. Columns should sum
# to ~1.0 (the code renormalizes if not).
# Files matching none of the globs are EXCLUDED from training.
CURRICULUM_SOURCES = [
("tinystories", ["tinystories*.txt", "tiny_*.txt"],
0.70, 0.10, 0.03, 0.02),
("simplewiki", ["simplewiki*.txt", "wiki_*.txt"],
0.10, 0.70, 0.10, 0.03),
("cosmopedia", ["cosmopedia*.txt", "cosmo_*.txt"],
0.10, 0.10, 0.70, 0.10),
("finephrase", ["finephrase*.txt", "fine_*.txt"],
0.10, 0.10, 0.17, 0.85),
]
# Hard floor: no source ever drops below this fraction at any step.
# 0.0 disables. 0.02 = 2% minimum, prevents catastrophic forgetting.
CURRICULUM_FLOOR = 0.02
mod_capacity = 0.25
use_mtp = False
mtp_depth = 1
mtp_lambda = 0.1
use_nvfp4 = False
# config.py
use_fp8 = False
fp8_recipe = "rowwise_with_gw_hp" # default
fp8_bf16_tail_frac = 0.15 # default
use_int8 = False
int8_group_size = 128
mod_budget_coeff = 0.05
mod_gate_entropy_coeff = 0.0
mod_hard_eval = False
use_flexattention = True
numberoflayers = 16
numberofheads = 12
D_MODEL = 384
ALPHA_COMPILE_MODE="reduce-overhead"
num_kv_heads = 6
sliding_window_size = 512
use_sdpa=False
optimizer_8bit = True
use_wsd = True
wsd_decay_steps = 3500
ffn_mult = 8 // 3
use_moe = False
n_experts = 8
n_shared = 1
# T4 path
PRECISION = 'bf16'
GLOBAL_DTYPE = PRECISION
USE_ASYNC_LAYER_PREFETCH = True
intra_op_threads = 2
num_workers = 2
GPU_SIDE_BATCHING = True
DISABLE_MMAP = False
batch_size = 320
weight_decay = 0.01
use_static_lr = False
static_lr = 1e-3
learning_rate = 1e-3
attn_gate_window = 16
max_steps = 19000
data_parallel_only = True
use_fsdp = False
dropout = 0.0
use_muon = True
DATA_PATH = './training_data'
data_path = DATA_PATH
TOKENIZER_MODEL_PATH = './tokenizer'
use_streaming_batcher = False
|