| # Replication config, scale "s" (~15M params). Mirrors upstream config.yaml | |
| # (commit 201852d) with sizes scaled down. Every deviation from upstream is | |
| # marked SCALED or CHANGED with the upstream value in brackets. | |
| model: | |
| d_model: 256 # SCALED [512] | |
| n_head: 4 # SCALED [8] | |
| d_ff: 704 # SCALED [1408] (~2.75 x d_model, same ratio) | |
| n_prelude: 2 | |
| n_recur: 1 | |
| n_coda: 0 | |
| max_steps: 8 # SCALED [24] | |
| min_steps: 1 | |
| halt_prior: 0.216 # SCALED [0.072] mean depth 4.6 of 8 (upstream 13.9 of 24, same 0.58 ratio) | |
| halt_thresh: 0.9 | |
| ponder_beta: 0.01 | |
| bptt_window: 8 # SCALED [24] = max_steps, as upstream | |
| train_steps_mean: 4.3 # SCALED [12.8] same 0.53 share of max_steps | |
| context_start: 1024 # SCALED [2048] | |
| context_end: 1024 # CHANGED [4096] context growth off: not part of any claim under test | |
| context_step: 1 | |
| context_grow_every_chars: 100_000 | |
| context_gain_min: 0.015 | |
| context_every_chars: 65536 | |
| pool: | |
| experts: 32 # SCALED [64] | |
| width: 512 # SCALED [2048] | |
| depth: 1 | |
| top_k: 4 # SCALED [8] same 25% of resident | |
| resident: 16 # SCALED [32] same 50% of starting pool | |
| ram_cache: 96 | |
| capacity_factor: 1.5 | |
| segment_chars: 2048 | |
| reselect_chars: 64 | |
| margin: 0.10 | |
| dwell_chars: 512 # SCALED [2048] = one chunk, as upstream | |
| explore: 0.15 | |
| training: | |
| save_every: 5 | |
| precision: bf16 | |
| chunk: 512 # SCALED [2048] same chunk:context ratio of 1:2 as upstream at 4096 | |
| lr: 3.0e-4 | |
| trunk_lr_mult: 0.1 | |
| weight_decay: 0.1 | |
| clip: 1.0 | |
| growth: | |
| every_chars: 500_000 # SCALED [2M] run is ~4x shorter than upstream's 409M | |
| k: 1 | |
| max_gap: 0.4 | |
| dying_frac_max: 0.35 | |
| max_in_flight: 50 | |
| keep_ratio_min: 0.35 | |
| recent_mult: 4.0 | |
| max_disk_gb: 10.0 | |
| mem_frac: 0.95 | |
| birth_gate: 0.001 | |
| prune: | |
| survival_chars: 25_000_000 # SCALED [100M] same 4x | |
| dying_at: 0.65 | |
| data: | |
| train: upstream/mini-AGI/data/train | |
| held_out: upstream/mini-AGI/data/val | |
| weights: runs/rep_s/weights | |
| shuffle_seed: 0 | |
| passage: 32768 | |
| decoding: | |
| adapt_strength: 2.5 | |
| adapt_decay: 0.88 | |
| rep_penalty: 1.0 | |
| plots: | |
| enabled: false # CHANGED: upstream's plot tool is not in the repo | |
| last: 100 | |