| # Training settings, re-read at every (re)start of the training loop. | |
| # To change them without giving up the nodes: edit this file, then `touch logs/RELOAD`. | |
| BIN="/fsx/ajay-stu/ar-data/bin/fineweb.bin /fsx/ajay-stu/ar-data/bin/fw003/fineweb.bin /fsx/ajay-stu/ar-data/bin/fw004/fineweb.bin /fsx/ajay-stu/ar-data/bin/fw005/fineweb.bin /fsx/ajay-stu/ar-data/bin/fw006/fineweb.bin /fsx/ajay-stu/ar-data/bin/fw007/fineweb.bin /fsx/ajay-stu/ar-data/bin/fw008/fineweb.bin /fsx/ajay-stu/ar-data/bin/fw009/fineweb.bin /fsx/ajay-stu/ar-data/bin/stack_python.bin /fsx/ajay-stu/ar-data/bin/stack_javascript.bin /fsx/ajay-stu/ar-data/bin/stack_java.bin /fsx/ajay-stu/ar-data/bin/stack_cpp.bin /fsx/ajay-stu/ar-data/bin/stack_typescript.bin /fsx/ajay-stu/ar-data/bin/stack_go.bin /fsx/ajay-stu/ar-data/bin/stack_rust.bin /fsx/ajay-stu/ar-data/bin/stack_shell.bin /fsx/ajay-stu/ar-data/bin/stack_sql.bin /fsx/ajay-stu/ar-data/bin/finemath.bin /fsx/ajay-stu/ar-data/bin/finemath3.bin" | |
| WEIGHTS="0.174428 0.067939 0.067939 0.067939 0.067939 0.067939 0.067939 0.067939 0.070791 0.033565 0.033565 0.023975 0.019180 0.004795 0.004795 0.004539 0.004795 0.044444 0.105556" | |
| STOP_AT=619700 | |
| LR_HORIZON=572000 | |
| PEAK_LR=3e-4 | |
| # >>> stage2 (written by s2_prepare.py): Stage 2 switches on by itself at step 572,000 | |
| export STAGE2_START=572000 STAGE2_WARMUP=1000 STAGE2_PEAK=1e-4 STAGE2_MIN=1e-5 | |
| export PROTECT_CKPTS="ckpt_571999" | |
| export S2_BIN="/fsx/ajay-stu/ar-data/stage2/hq010/fineweb_hq.bin /fsx/ajay-stu/ar-data/stage2/hq011/fineweb_hq.bin /fsx/ajay-stu/ar-data/stage2/hq012/fineweb_hq.bin /fsx/ajay-stu/ar-data/stage2/cosmopedia.bin /fsx/ajay-stu/ar-data/bin/stack_python.bin /fsx/ajay-stu/ar-data/bin/stack_javascript.bin /fsx/ajay-stu/ar-data/bin/stack_java.bin /fsx/ajay-stu/ar-data/bin/stack_cpp.bin /fsx/ajay-stu/ar-data/bin/stack_typescript.bin /fsx/ajay-stu/ar-data/bin/stack_go.bin /fsx/ajay-stu/ar-data/bin/stack_rust.bin /fsx/ajay-stu/ar-data/bin/stack_shell.bin /fsx/ajay-stu/ar-data/bin/stack_sql.bin /fsx/ajay-stu/ar-data/bin/finemath.bin /fsx/ajay-stu/ar-data/stage2/infiwebmath.bin /fsx/ajay-stu/ar-data/bin/finemath3.bin" | |
| export S2_WEIGHTS="0.066667 0.066667 0.066667 0.100000 0.106186 0.050347 0.050347 0.035962 0.028770 0.007192 0.007192 0.006809 0.007192 0.160000 0.120000 0.120000" | |
| # <<< stage2 | |