File size: 2,270 Bytes
0662d8e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 | # Replication config, scale "s" (~15M params). Mirrors upstream config.yaml
# (commit 201852d) with sizes scaled down. Every deviation from upstream is
# marked SCALED or CHANGED with the upstream value in brackets.
model:
d_model: 256 # SCALED [512]
n_head: 4 # SCALED [8]
d_ff: 704 # SCALED [1408] (~2.75 x d_model, same ratio)
n_prelude: 2
n_recur: 1
n_coda: 0
max_steps: 8 # SCALED [24]
min_steps: 1
halt_prior: 0.216 # SCALED [0.072] mean depth 4.6 of 8 (upstream 13.9 of 24, same 0.58 ratio)
halt_thresh: 0.9
ponder_beta: 0.01
bptt_window: 8 # SCALED [24] = max_steps, as upstream
train_steps_mean: 4.3 # SCALED [12.8] same 0.53 share of max_steps
context_start: 1024 # SCALED [2048]
context_end: 1024 # CHANGED [4096] context growth off: not part of any claim under test
context_step: 1
context_grow_every_chars: 100_000
context_gain_min: 0.015
context_every_chars: 65536
pool:
experts: 32 # SCALED [64]
width: 512 # SCALED [2048]
depth: 1
top_k: 4 # SCALED [8] same 25% of resident
resident: 16 # SCALED [32] same 50% of starting pool
ram_cache: 96
capacity_factor: 1.5
segment_chars: 2048
reselect_chars: 64
margin: 0.10
dwell_chars: 512 # SCALED [2048] = one chunk, as upstream
explore: 0.15
training:
save_every: 5
precision: bf16
chunk: 512 # SCALED [2048] same chunk:context ratio of 1:2 as upstream at 4096
lr: 3.0e-4
trunk_lr_mult: 0.1
weight_decay: 0.1
clip: 1.0
growth:
every_chars: 500_000 # SCALED [2M] run is ~4x shorter than upstream's 409M
k: 1
max_gap: 0.4
dying_frac_max: 0.35
max_in_flight: 50
keep_ratio_min: 0.35
recent_mult: 4.0
max_disk_gb: 10.0
mem_frac: 0.95
birth_gate: 0.001
prune:
survival_chars: 25_000_000 # SCALED [100M] same 4x
dying_at: 0.65
data:
train: upstream/mini-AGI/data/train
held_out: upstream/mini-AGI/data/val
weights: runs/rep_s/weights
shuffle_seed: 0
passage: 32768
decoding:
adapt_strength: 2.5
adapt_decay: 0.88
rep_penalty: 1.0
plots:
enabled: false # CHANGED: upstream's plot tool is not in the repo
last: 100
|