diff --git a/GQA/run/SHA256SUMS_kopiarka b/GQA/run/SHA256SUMS_kopiarka new file mode 100644 index 0000000000000000000000000000000000000000..0d633fbd47f48acbc76addc3b2191837f4717d63 --- /dev/null +++ b/GQA/run/SHA256SUMS_kopiarka @@ -0,0 +1,5 @@ +8296514387f2f1fbc20f5c61822007daa823f86b541dc64e830b1361358fba6a ckpt_500.pt +fb4a4a805f6e88204a3a859238567ff28f40c9f0adbf5a8fdf6c07119245dfec ckpt_1000.pt +13ddaf2ea734dd720aaa3c55807621d3b66f44ce4d1a58d782970491fafc8699 ckpt_1500.pt +e8e632635f4bca6f8d4885502d46503c8ac16f2e83d5d1aa65129535215f828e ckpt_2000.pt +3366e88771415a893cb14f3d209b7759bde43b7475e594bf8a81a88c68843f2f ckpt_2500.pt diff --git a/GQA/run/args.txt b/GQA/run/args.txt new file mode 100644 index 0000000000000000000000000000000000000000..9d7f394985773d50cee1840de3df701642621029 --- /dev/null +++ b/GQA/run/args.txt @@ -0,0 +1 @@ +--data-dir /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2 --out-dir /home/ubuntu/kolektyw/dowody400m/wyniki/GQA --run-id dowody_GQA --vocab 32768 --dtype uint16 --block 4096 --n-layer 18 --n-embd 896 --n-head 14 --kv-heads 2 --value-residual --batch 128 --grad-accum 8 --norm rmsnorm --pos rope --rope-theta 500000 --ffn swiglu --ffn-mult 2.667 --qk-norm --attn-bias 0 --optimizer muon --muon-lr 0.02 --lr 6e-4 --min-lr 6e-05 --wd 0.1 --schedule wsd --decay-frac 0.2 --warmup 500 --act-ckpt --loss-chunk 1024 --sampler random --seed 1 --data-seed 20261022 --compile --log-every 20 --eval-every 4769 --eval-iters 1 --steps 4768 --ckpt-every 500 \ No newline at end of file diff --git a/GQA/run/metrics.jsonl b/GQA/run/metrics.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..95a57006e85845773b75974543b992f512301bb0 --- /dev/null +++ b/GQA/run/metrics.jsonl @@ -0,0 +1,146 @@ +{"step": 20, "loss": 10.382979106903075, "lr": 2.3999999999999997e-05, "tok_s": 40037.62144840001} +{"step": 40, "loss": 9.56647868156433, "lr": 4.7999999999999994e-05, "tok_s": 55950.47213670447} +{"step": 60, "loss": 8.688220500946045, "lr": 7.199999999999999e-05, "tok_s": 55956.14811527882} +{"step": 80, "loss": 8.038008499145509, "lr": 9.599999999999999e-05, "tok_s": 55949.9778076849} +{"step": 100, "loss": 7.590619421005249, "lr": 0.00011999999999999999, "tok_s": 55935.00502426609} +{"step": 120, "loss": 7.095160555839539, "lr": 0.00014399999999999998, "tok_s": 55916.81302638947} +{"step": 140, "loss": 6.665168881416321, "lr": 0.000168, "tok_s": 55909.52022217888} +{"step": 160, "loss": 6.325761508941651, "lr": 0.00019199999999999998, "tok_s": 55911.599787190855} +{"step": 180, "loss": 6.088504076004028, "lr": 0.00021599999999999996, "tok_s": 55907.69744272099} +{"step": 200, "loss": 5.892195367813111, "lr": 0.00023999999999999998, "tok_s": 55900.68233984899} +{"step": 220, "loss": 5.719494819641113, "lr": 0.00026399999999999997, "tok_s": 55896.15593229689} +{"step": 240, "loss": 5.564488887786865, "lr": 0.00028799999999999995, "tok_s": 55896.514616477696} +{"step": 260, "loss": 5.411128854751587, "lr": 0.000312, "tok_s": 55891.311766931656} +{"step": 280, "loss": 5.264648008346557, "lr": 0.000336, "tok_s": 55890.142672656504} +{"step": 300, "loss": 5.125132918357849, "lr": 0.00035999999999999997, "tok_s": 55889.63648279308} +{"step": 320, "loss": 4.998163890838623, "lr": 0.00038399999999999996, "tok_s": 55889.25302894563} +{"step": 340, "loss": 4.875051689147949, "lr": 0.000408, "tok_s": 55887.41942666463} +{"step": 360, "loss": 4.771002888679504, "lr": 0.00043199999999999993, "tok_s": 55884.27258425807} +{"step": 380, "loss": 4.655315661430359, "lr": 0.00045599999999999997, "tok_s": 55884.96934286311} +{"step": 400, "loss": 4.5465178966522215, "lr": 0.00047999999999999996, "tok_s": 55881.25837321708} +{"step": 420, "loss": 4.448748826980591, "lr": 0.000504, "tok_s": 55880.59188239282} +{"step": 440, "loss": 4.359668445587158, "lr": 0.0005279999999999999, "tok_s": 55881.40222428677} +{"step": 460, "loss": 4.275037050247192, "lr": 0.000552, "tok_s": 55884.09150939727} +{"step": 480, "loss": 4.2145402193069454, "lr": 0.0005759999999999999, "tok_s": 55873.46532254367} +{"step": 500, "loss": 4.149071002006531, "lr": 0.0006, "tok_s": 55877.22857218452} +{"step": 520, "loss": 4.10716700553894, "lr": 0.0006, "tok_s": 55403.22907999532} +{"step": 540, "loss": 4.052633845806122, "lr": 0.0006, "tok_s": 55892.173277238464} +{"step": 560, "loss": 3.9963469982147215, "lr": 0.0006, "tok_s": 55878.88699638042} +{"step": 580, "loss": 3.9511124730110168, "lr": 0.0006, "tok_s": 55869.96262589284} +{"step": 600, "loss": 3.9063917636871337, "lr": 0.0006, "tok_s": 55865.86760636415} +{"step": 620, "loss": 3.8814687967300414, "lr": 0.0006, "tok_s": 55861.46000312544} +{"step": 640, "loss": 3.8532938361167908, "lr": 0.0006, "tok_s": 55856.9303679642} +{"step": 660, "loss": 3.821173572540283, "lr": 0.0006, "tok_s": 55853.595086182206} +{"step": 680, "loss": 3.795495903491974, "lr": 0.0006, "tok_s": 55851.20208356764} +{"step": 700, "loss": 3.76346892118454, "lr": 0.0006, "tok_s": 55864.89045999702} +{"step": 720, "loss": 3.731121325492859, "lr": 0.0006, "tok_s": 55892.40952421342} +{"step": 740, "loss": 3.7248404145240785, "lr": 0.0006, "tok_s": 55852.72050783114} +{"step": 760, "loss": 3.6998636722564697, "lr": 0.0006, "tok_s": 55854.804361771116} +{"step": 780, "loss": 3.673145031929016, "lr": 0.0006, "tok_s": 55852.95947141054} +{"step": 800, "loss": 3.6588367223739624, "lr": 0.0006, "tok_s": 55854.218443901394} +{"step": 820, "loss": 3.6433533906936644, "lr": 0.0006, "tok_s": 55852.26386539599} +{"step": 840, "loss": 3.627117228507996, "lr": 0.0006, "tok_s": 55855.40802608909} +{"step": 860, "loss": 3.6044566869735717, "lr": 0.0006, "tok_s": 55855.47066321617} +{"step": 880, "loss": 3.598527717590332, "lr": 0.0006, "tok_s": 55857.019824091876} +{"step": 900, "loss": 3.580390286445618, "lr": 0.0006, "tok_s": 55852.60659477567} +{"step": 920, "loss": 3.5707054495811463, "lr": 0.0006, "tok_s": 55850.399922464516} +{"step": 940, "loss": 3.5649645924568176, "lr": 0.0006, "tok_s": 55852.27450469675} +{"step": 960, "loss": 3.5389197945594786, "lr": 0.0006, "tok_s": 55853.04763806798} +{"step": 980, "loss": 3.5352579712867738, "lr": 0.0006, "tok_s": 55853.98124287446} +{"step": 1000, "loss": 3.523323690891266, "lr": 0.0006, "tok_s": 55851.88170440031} +{"step": 1020, "loss": 3.5180077075958254, "lr": 0.0006, "tok_s": 55193.14358495176} +{"step": 1040, "loss": 3.4907343983650208, "lr": 0.0006, "tok_s": 55861.28688074019} +{"step": 1060, "loss": 3.501312184333801, "lr": 0.0006, "tok_s": 55862.68153976331} +{"step": 1080, "loss": 3.490531289577484, "lr": 0.0006, "tok_s": 55864.21165788537} +{"step": 1100, "loss": 3.474697434902191, "lr": 0.0006, "tok_s": 55862.198694343766} +{"step": 1120, "loss": 3.4564267992973328, "lr": 0.0006, "tok_s": 55860.18296696393} +{"step": 1140, "loss": 3.4498842358589172, "lr": 0.0006, "tok_s": 55859.96075629045} +{"step": 1160, "loss": 3.4431535243988036, "lr": 0.0006, "tok_s": 55860.42667706288} +{"step": 1180, "loss": 3.4448654890060424, "lr": 0.0006, "tok_s": 55865.909900471794} +{"step": 1200, "loss": 3.42562221288681, "lr": 0.0006, "tok_s": 55869.89520103725} +{"step": 1220, "loss": 3.423185706138611, "lr": 0.0006, "tok_s": 55867.896305433605} +{"step": 1240, "loss": 3.414616274833679, "lr": 0.0006, "tok_s": 55858.96210715924} +{"step": 1260, "loss": 3.3998213529586794, "lr": 0.0006, "tok_s": 55863.03539518924} +{"step": 1280, "loss": 3.402288496494293, "lr": 0.0006, "tok_s": 55864.79679198819} +{"step": 1300, "loss": 3.3960714101791383, "lr": 0.0006, "tok_s": 55864.42751601959} +{"step": 1320, "loss": 3.3870596051216126, "lr": 0.0006, "tok_s": 55858.42377658899} +{"step": 1340, "loss": 3.3868780732154846, "lr": 0.0006, "tok_s": 55852.23634509015} +{"step": 1360, "loss": 3.3778282761573792, "lr": 0.0006, "tok_s": 55851.77190855575} +{"step": 1380, "loss": 3.369214880466461, "lr": 0.0006, "tok_s": 55856.39157998008} +{"step": 1400, "loss": 3.3584998488426208, "lr": 0.0006, "tok_s": 55858.81000005289} +{"step": 1420, "loss": 3.3467327117919923, "lr": 0.0006, "tok_s": 55856.025110774484} +{"step": 1440, "loss": 3.35287891626358, "lr": 0.0006, "tok_s": 55860.86216782732} +{"step": 1460, "loss": 3.3505175709724426, "lr": 0.0006, "tok_s": 55860.247885166784} +{"step": 1480, "loss": 3.354390799999237, "lr": 0.0006, "tok_s": 55859.782960645454} +{"step": 1500, "loss": 3.3270774483680725, "lr": 0.0006, "tok_s": 55856.53941754428} +{"step": 1520, "loss": 3.3344152092933657, "lr": 0.0006, "tok_s": 55159.741563164695} +{"step": 1540, "loss": 3.3191637277603148, "lr": 0.0006, "tok_s": 55858.31296180528} +{"step": 1560, "loss": 3.3265525102615356, "lr": 0.0006, "tok_s": 55868.37108785107} +{"step": 1580, "loss": 3.3150747060775756, "lr": 0.0006, "tok_s": 55867.44062125878} +{"step": 1600, "loss": 3.304818665981293, "lr": 0.0006, "tok_s": 55866.26422258368} +{"step": 1620, "loss": 3.304558289051056, "lr": 0.0006, "tok_s": 55863.76383918023} +{"step": 1640, "loss": 3.30103884935379, "lr": 0.0006, "tok_s": 55858.05465581214} +{"step": 1660, "loss": 3.3019826889038084, "lr": 0.0006, "tok_s": 55855.77065656859} +{"step": 1680, "loss": 3.290108013153076, "lr": 0.0006, "tok_s": 55858.40660807259} +{"step": 1700, "loss": 3.2835023045539855, "lr": 0.0006, "tok_s": 55858.18043868478} +{"step": 1720, "loss": 3.285639023780823, "lr": 0.0006, "tok_s": 55861.454326964646} +{"step": 1740, "loss": 3.278670835494995, "lr": 0.0006, "tok_s": 55859.09633669302} +{"step": 1760, "loss": 3.2777636289596557, "lr": 0.0006, "tok_s": 55858.65094117567} +{"step": 1780, "loss": 3.265216624736786, "lr": 0.0006, "tok_s": 55855.48868117003} +{"step": 1800, "loss": 3.2684202671051024, "lr": 0.0006, "tok_s": 55849.78998337803} +{"step": 1820, "loss": 3.2644393920898436, "lr": 0.0006, "tok_s": 55852.98990057944} +{"step": 1840, "loss": 3.2582288146018983, "lr": 0.0006, "tok_s": 55857.068773244166} +{"step": 1860, "loss": 3.257483172416687, "lr": 0.0006, "tok_s": 55858.88364135634} +{"step": 1880, "loss": 3.252480161190033, "lr": 0.0006, "tok_s": 55860.95596454378} +{"step": 1900, "loss": 3.2468705415725707, "lr": 0.0006, "tok_s": 55868.200335577094} +{"step": 1920, "loss": 3.2458807229995728, "lr": 0.0006, "tok_s": 55866.75948627773} +{"step": 1940, "loss": 3.2395505785942076, "lr": 0.0006, "tok_s": 55861.67356951348} +{"step": 1960, "loss": 3.2437692284584045, "lr": 0.0006, "tok_s": 55861.859749650495} +{"step": 1980, "loss": 3.2375988006591796, "lr": 0.0006, "tok_s": 55860.48655841409} +{"step": 2000, "loss": 3.23126403093338, "lr": 0.0006, "tok_s": 55857.7651374327} +{"step": 2020, "loss": 3.2307679533958433, "lr": 0.0006, "tok_s": 55181.14931078551} +{"step": 2040, "loss": 3.2255444169044494, "lr": 0.0006, "tok_s": 55855.61686435047} +{"step": 2060, "loss": 3.225483739376068, "lr": 0.0006, "tok_s": 55853.64168857098} +{"step": 2080, "loss": 3.213956689834595, "lr": 0.0006, "tok_s": 55850.38105670706} +{"step": 2100, "loss": 3.222394919395447, "lr": 0.0006, "tok_s": 55847.23533444284} +{"step": 2120, "loss": 3.210958731174469, "lr": 0.0006, "tok_s": 55849.87991309439} +{"step": 2140, "loss": 3.2240227699279784, "lr": 0.0006, "tok_s": 55844.74197226356} +{"step": 2160, "loss": 3.2026231408119203, "lr": 0.0006, "tok_s": 55842.073205979555} +{"step": 2180, "loss": 3.204519212245941, "lr": 0.0006, "tok_s": 55843.07706555174} +{"step": 2200, "loss": 3.2058327913284304, "lr": 0.0006, "tok_s": 55847.001525742235} +{"step": 2220, "loss": 3.1977394461631774, "lr": 0.0006, "tok_s": 55847.026629767315} +{"step": 2240, "loss": 3.194538748264313, "lr": 0.0006, "tok_s": 55851.65757380692} +{"step": 2260, "loss": 3.191618466377258, "lr": 0.0006, "tok_s": 55853.358883489906} +{"step": 2280, "loss": 3.1867557644844053, "lr": 0.0006, "tok_s": 55856.43776140001} +{"step": 2300, "loss": 3.190177059173584, "lr": 0.0006, "tok_s": 55851.96979672725} +{"step": 2320, "loss": 3.1867308974266053, "lr": 0.0006, "tok_s": 55851.54799162408} +{"step": 2340, "loss": 3.182142972946167, "lr": 0.0006, "tok_s": 55849.37934584578} +{"step": 2360, "loss": 3.172446036338806, "lr": 0.0006, "tok_s": 55850.881570987054} +{"step": 2380, "loss": 3.171089839935303, "lr": 0.0006, "tok_s": 55851.00037086171} +{"step": 2400, "loss": 3.1715157270431518, "lr": 0.0006, "tok_s": 55857.74314522562} +{"step": 2420, "loss": 3.173757755756378, "lr": 0.0006, "tok_s": 55855.25501616669} +{"step": 2440, "loss": 3.168051612377167, "lr": 0.0006, "tok_s": 55852.09030350863} +{"step": 2460, "loss": 3.167601001262665, "lr": 0.0006, "tok_s": 55857.334306155106} +{"step": 2480, "loss": 3.1639285683631897, "lr": 0.0006, "tok_s": 55854.32874590251} +{"step": 2500, "loss": 3.159174644947052, "lr": 0.0006, "tok_s": 55855.53521578178} +{"step": 2520, "loss": 3.1603678584098818, "lr": 0.0006, "tok_s": 55177.35615187041} +{"step": 2540, "loss": 3.1611927270889284, "lr": 0.0006, "tok_s": 55851.00540656905} +{"step": 2560, "loss": 3.1577532649040223, "lr": 0.0006, "tok_s": 55853.75659890297} +{"step": 2580, "loss": 3.1542645573616026, "lr": 0.0006, "tok_s": 55855.57040053972} +{"step": 2600, "loss": 3.144371044635773, "lr": 0.0006, "tok_s": 55855.47924657069} +{"step": 2620, "loss": 3.15311598777771, "lr": 0.0006, "tok_s": 55850.350559456936} +{"step": 2640, "loss": 3.153462862968445, "lr": 0.0006, "tok_s": 55850.68312312965} +{"step": 2660, "loss": 3.152262246608734, "lr": 0.0006, "tok_s": 55850.7300043583} +{"step": 2680, "loss": 3.1478206515312195, "lr": 0.0006, "tok_s": 55850.78298520614} +{"step": 2700, "loss": 3.1494753122329713, "lr": 0.0006, "tok_s": 55848.569010462445} +{"step": 2720, "loss": 3.132279109954834, "lr": 0.0006, "tok_s": 55854.512323446455} +{"step": 2740, "loss": 3.1343593835830688, "lr": 0.0006, "tok_s": 55856.64653642186} +{"step": 2760, "loss": 3.1370200872421266, "lr": 0.0006, "tok_s": 55859.93840761366} +{"step": 2780, "loss": 3.12802734375, "lr": 0.0006, "tok_s": 55860.464351215545} +{"step": 2800, "loss": 3.123040521144867, "lr": 0.0006, "tok_s": 55865.237743976984} +{"step": 2820, "loss": 3.1200297117233275, "lr": 0.0006, "tok_s": 55862.59291695877} +{"step": 2840, "loss": 3.128438580036163, "lr": 0.0006, "tok_s": 55856.25268148453} +{"step": 2860, "loss": 3.120231294631958, "lr": 0.0006, "tok_s": 55853.544440752} +{"step": 2880, "loss": 3.122507381439209, "lr": 0.0006, "tok_s": 55854.86458574521} +{"step": 2900, "loss": 3.123218131065369, "lr": 0.0006, "tok_s": 55852.98663777653} +{"step": 2920, "loss": 3.1307594418525695, "lr": 0.0006, "tok_s": 55857.744989732644} diff --git a/GQA/run/srodowisko_GQA_mini0_resume0.txt b/GQA/run/srodowisko_GQA_mini0_resume0.txt new file mode 100644 index 0000000000000000000000000000000000000000..604c1814b782247dfbd4b831dd7bf8614cba0bf8 --- /dev/null +++ b/GQA/run/srodowisko_GQA_mini0_resume0.txt @@ -0,0 +1,64 @@ +data 2026-10-23T02:33:05Z ARM GQA GPU 2 MINI 0 DATA_SEED 20261022 CKPT 500 RESUME 0 DEST /var/lib/docker/kolektyw/dowody400m_ckpt/wyniki_GQA +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +0702c32c0ad602d8d12b16539e05cfbc68d1823cfd39b82b984357d78f27f664 /home/ubuntu/kolektyw/dowody400m/kod/ramie_v03.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +341aae81ce8512294dcd7de93ced26053025677b4ab449606be989f7fd898d27 /home/ubuntu/kolektyw/dowody400m/kod/kopiarka_ckpt.sh +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 diff --git a/GQA/run/srodowisko_resume0.txt b/GQA/run/srodowisko_resume0.txt new file mode 100644 index 0000000000000000000000000000000000000000..604c1814b782247dfbd4b831dd7bf8614cba0bf8 --- /dev/null +++ b/GQA/run/srodowisko_resume0.txt @@ -0,0 +1,64 @@ +data 2026-10-23T02:33:05Z ARM GQA GPU 2 MINI 0 DATA_SEED 20261022 CKPT 500 RESUME 0 DEST /var/lib/docker/kolektyw/dowody400m_ckpt/wyniki_GQA +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +0702c32c0ad602d8d12b16539e05cfbc68d1823cfd39b82b984357d78f27f664 /home/ubuntu/kolektyw/dowody400m/kod/ramie_v03.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +341aae81ce8512294dcd7de93ced26053025677b4ab449606be989f7fd898d27 /home/ubuntu/kolektyw/dowody400m/kod/kopiarka_ckpt.sh +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 diff --git a/GQA/run/train.log b/GQA/run/train.log new file mode 100644 index 0000000000000000000000000000000000000000..b762f3c7db2395d7244b16fb664e543f639195ba --- /dev/null +++ b/GQA/run/train.log @@ -0,0 +1,159 @@ +[04:33:11] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[04:33:11] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[04:33:13] model GPT-ref: 178.1M param (L18 d896 h14 kv2) +[04:33:14] optimizer=muon muon_lr=0.02 (mult=33.3x) +[04:33:14] r11 pamiec: act_ckpt=True loss_chunk=1024 +[04:33:14] r13: mixer_layout=AAAAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[04:33:14] torch.compile enabled +[04:33:14] START gpt-ref: steps=4768 (od 0) lr=0.0006->6e-05 +[04:37:36] step 20/4768 loss 10.3830 lr 2.40e-05 gnorm 2.93 40,038 tok/s peakVRAM 13.3GB +[04:40:44] step 40/4768 loss 9.5665 lr 4.80e-05 gnorm 1.43 55,950 tok/s peakVRAM 13.3GB +[04:43:51] step 60/4768 loss 8.6882 lr 7.20e-05 gnorm 0.83 55,956 tok/s peakVRAM 13.3GB +[04:46:58] step 80/4768 loss 8.0380 lr 9.60e-05 gnorm 1.30 55,950 tok/s peakVRAM 13.3GB +[04:50:06] step 100/4768 loss 7.5906 lr 1.20e-04 gnorm 1.02 55,935 tok/s peakVRAM 13.3GB +[04:53:13] step 120/4768 loss 7.0952 lr 1.44e-04 gnorm 0.93 55,917 tok/s peakVRAM 13.3GB +[04:56:21] step 140/4768 loss 6.6652 lr 1.68e-04 gnorm 0.70 55,910 tok/s peakVRAM 13.3GB +[04:59:28] step 160/4768 loss 6.3258 lr 1.92e-04 gnorm 0.73 55,912 tok/s peakVRAM 13.3GB +[05:02:36] step 180/4768 loss 6.0885 lr 2.16e-04 gnorm 0.78 55,908 tok/s peakVRAM 13.3GB +[05:05:44] step 200/4768 loss 5.8922 lr 2.40e-04 gnorm 0.68 55,901 tok/s peakVRAM 13.3GB +[05:08:51] step 220/4768 loss 5.7195 lr 2.64e-04 gnorm 0.78 55,896 tok/s peakVRAM 13.3GB +[05:11:59] step 240/4768 loss 5.5645 lr 2.88e-04 gnorm 0.84 55,897 tok/s peakVRAM 13.3GB +[05:15:06] step 260/4768 loss 5.4111 lr 3.12e-04 gnorm 0.72 55,891 tok/s peakVRAM 13.3GB +[05:18:14] step 280/4768 loss 5.2646 lr 3.36e-04 gnorm 0.95 55,890 tok/s peakVRAM 13.3GB +[05:21:22] step 300/4768 loss 5.1251 lr 3.60e-04 gnorm 0.78 55,890 tok/s peakVRAM 13.3GB +[05:24:29] step 320/4768 loss 4.9982 lr 3.84e-04 gnorm 0.77 55,889 tok/s peakVRAM 13.3GB +[05:27:37] step 340/4768 loss 4.8751 lr 4.08e-04 gnorm 0.92 55,887 tok/s peakVRAM 13.3GB +[05:30:44] step 360/4768 loss 4.7710 lr 4.32e-04 gnorm 0.59 55,884 tok/s peakVRAM 13.3GB +[05:33:52] step 380/4768 loss 4.6553 lr 4.56e-04 gnorm 0.80 55,885 tok/s peakVRAM 13.3GB +[05:37:00] step 400/4768 loss 4.5465 lr 4.80e-04 gnorm 0.86 55,881 tok/s peakVRAM 13.3GB +[05:40:07] step 420/4768 loss 4.4487 lr 5.04e-04 gnorm 0.73 55,881 tok/s peakVRAM 13.3GB +[05:43:15] step 440/4768 loss 4.3597 lr 5.28e-04 gnorm 0.71 55,881 tok/s peakVRAM 13.3GB +[05:46:23] step 460/4768 loss 4.2750 lr 5.52e-04 gnorm 0.74 55,884 tok/s peakVRAM 13.3GB +[05:49:30] step 480/4768 loss 4.2145 lr 5.76e-04 gnorm 0.55 55,873 tok/s peakVRAM 13.3GB +[05:52:38] step 500/4768 loss 4.1491 lr 6.00e-04 gnorm 0.72 55,877 tok/s peakVRAM 13.3GB +[05:52:40] ckpt @ 500 +[05:55:47] step 520/4768 loss 4.1072 lr 6.00e-04 gnorm 0.50 55,403 tok/s peakVRAM 13.3GB +[05:58:55] step 540/4768 loss 4.0526 lr 6.00e-04 gnorm 0.56 55,892 tok/s peakVRAM 13.3GB +[06:02:03] step 560/4768 loss 3.9963 lr 6.00e-04 gnorm 0.45 55,879 tok/s peakVRAM 13.3GB +[06:05:10] step 580/4768 loss 3.9511 lr 6.00e-04 gnorm 0.48 55,870 tok/s peakVRAM 13.3GB +[06:08:18] step 600/4768 loss 3.9064 lr 6.00e-04 gnorm 0.47 55,866 tok/s peakVRAM 13.3GB +[06:11:26] step 620/4768 loss 3.8815 lr 6.00e-04 gnorm 0.41 55,861 tok/s peakVRAM 13.3GB +[06:14:33] step 640/4768 loss 3.8533 lr 6.00e-04 gnorm 0.38 55,857 tok/s peakVRAM 13.3GB +[06:17:41] step 660/4768 loss 3.8212 lr 6.00e-04 gnorm 0.31 55,854 tok/s peakVRAM 13.3GB +[06:20:49] step 680/4768 loss 3.7955 lr 6.00e-04 gnorm 0.35 55,851 tok/s peakVRAM 13.3GB +[06:23:57] step 700/4768 loss 3.7635 lr 6.00e-04 gnorm 0.50 55,865 tok/s peakVRAM 13.3GB +[06:27:04] step 720/4768 loss 3.7311 lr 6.00e-04 gnorm 0.39 55,892 tok/s peakVRAM 13.3GB +[06:30:12] step 740/4768 loss 3.7248 lr 6.00e-04 gnorm 0.31 55,853 tok/s peakVRAM 13.3GB +[06:33:20] step 760/4768 loss 3.6999 lr 6.00e-04 gnorm 0.39 55,855 tok/s peakVRAM 13.3GB +[06:36:27] step 780/4768 loss 3.6731 lr 6.00e-04 gnorm 0.32 55,853 tok/s peakVRAM 13.3GB +[06:39:35] step 800/4768 loss 3.6588 lr 6.00e-04 gnorm 0.30 55,854 tok/s peakVRAM 13.3GB +[06:42:43] step 820/4768 loss 3.6434 lr 6.00e-04 gnorm 0.33 55,852 tok/s peakVRAM 13.3GB +[06:45:51] step 840/4768 loss 3.6271 lr 6.00e-04 gnorm 0.34 55,855 tok/s peakVRAM 13.3GB +[06:48:58] step 860/4768 loss 3.6045 lr 6.00e-04 gnorm 0.26 55,855 tok/s peakVRAM 13.3GB +[06:52:06] step 880/4768 loss 3.5985 lr 6.00e-04 gnorm 0.24 55,857 tok/s peakVRAM 13.3GB +[06:55:14] step 900/4768 loss 3.5804 lr 6.00e-04 gnorm 0.36 55,853 tok/s peakVRAM 13.3GB +[06:58:22] step 920/4768 loss 3.5707 lr 6.00e-04 gnorm 0.28 55,850 tok/s peakVRAM 13.3GB +[07:01:29] step 940/4768 loss 3.5650 lr 6.00e-04 gnorm 0.26 55,852 tok/s peakVRAM 13.3GB +[07:04:37] step 960/4768 loss 3.5389 lr 6.00e-04 gnorm 0.27 55,853 tok/s peakVRAM 13.3GB +[07:07:45] step 980/4768 loss 3.5353 lr 6.00e-04 gnorm 0.22 55,854 tok/s peakVRAM 13.3GB +[07:10:52] step 1000/4768 loss 3.5233 lr 6.00e-04 gnorm 0.27 55,852 tok/s peakVRAM 13.3GB +[07:10:55] ckpt @ 1000 +[07:14:02] step 1020/4768 loss 3.5180 lr 6.00e-04 gnorm 0.25 55,193 tok/s peakVRAM 13.3GB +[07:17:10] step 1040/4768 loss 3.4907 lr 6.00e-04 gnorm 0.24 55,861 tok/s peakVRAM 13.3GB +[07:20:18] step 1060/4768 loss 3.5013 lr 6.00e-04 gnorm 0.23 55,863 tok/s peakVRAM 13.3GB +[07:23:26] step 1080/4768 loss 3.4905 lr 6.00e-04 gnorm 0.31 55,864 tok/s peakVRAM 13.3GB +[07:26:33] step 1100/4768 loss 3.4747 lr 6.00e-04 gnorm 0.25 55,862 tok/s peakVRAM 13.3GB +[07:29:41] step 1120/4768 loss 3.4564 lr 6.00e-04 gnorm 0.22 55,860 tok/s peakVRAM 13.3GB +[07:32:49] step 1140/4768 loss 3.4499 lr 6.00e-04 gnorm 0.26 55,860 tok/s peakVRAM 13.3GB +[07:35:56] step 1160/4768 loss 3.4432 lr 6.00e-04 gnorm 0.21 55,860 tok/s peakVRAM 13.3GB +[07:39:04] step 1180/4768 loss 3.4449 lr 6.00e-04 gnorm 0.23 55,866 tok/s peakVRAM 13.3GB +[07:42:12] step 1200/4768 loss 3.4256 lr 6.00e-04 gnorm 0.21 55,870 tok/s peakVRAM 13.3GB +[07:45:19] step 1220/4768 loss 3.4232 lr 6.00e-04 gnorm 0.24 55,868 tok/s peakVRAM 13.3GB +[07:48:27] step 1240/4768 loss 3.4146 lr 6.00e-04 gnorm 0.21 55,859 tok/s peakVRAM 13.3GB +[07:51:35] step 1260/4768 loss 3.3998 lr 6.00e-04 gnorm 0.21 55,863 tok/s peakVRAM 13.3GB +[07:54:43] step 1280/4768 loss 3.4023 lr 6.00e-04 gnorm 0.18 55,865 tok/s peakVRAM 13.3GB +[07:57:50] step 1300/4768 loss 3.3961 lr 6.00e-04 gnorm 0.25 55,864 tok/s peakVRAM 13.3GB +[08:00:58] step 1320/4768 loss 3.3871 lr 6.00e-04 gnorm 0.20 55,858 tok/s peakVRAM 13.3GB +[08:04:06] step 1340/4768 loss 3.3869 lr 6.00e-04 gnorm 0.21 55,852 tok/s peakVRAM 13.3GB +[08:07:14] step 1360/4768 loss 3.3778 lr 6.00e-04 gnorm 0.21 55,852 tok/s peakVRAM 13.3GB +[08:10:21] step 1380/4768 loss 3.3692 lr 6.00e-04 gnorm 0.20 55,856 tok/s peakVRAM 13.3GB +[08:13:29] step 1400/4768 loss 3.3585 lr 6.00e-04 gnorm 0.18 55,859 tok/s peakVRAM 13.3GB +[08:16:37] step 1420/4768 loss 3.3467 lr 6.00e-04 gnorm 0.21 55,856 tok/s peakVRAM 13.3GB +[08:19:44] step 1440/4768 loss 3.3529 lr 6.00e-04 gnorm 0.19 55,861 tok/s peakVRAM 13.3GB +[08:22:52] step 1460/4768 loss 3.3505 lr 6.00e-04 gnorm 0.18 55,860 tok/s peakVRAM 13.3GB +[08:26:00] step 1480/4768 loss 3.3544 lr 6.00e-04 gnorm 0.22 55,860 tok/s peakVRAM 13.3GB +[08:29:08] step 1500/4768 loss 3.3271 lr 6.00e-04 gnorm 0.18 55,857 tok/s peakVRAM 13.3GB +[08:29:10] ckpt @ 1500 +[08:32:18] step 1520/4768 loss 3.3344 lr 6.00e-04 gnorm 0.19 55,160 tok/s peakVRAM 13.3GB +[08:35:25] step 1540/4768 loss 3.3192 lr 6.00e-04 gnorm 0.23 55,858 tok/s peakVRAM 13.3GB +[08:38:33] step 1560/4768 loss 3.3266 lr 6.00e-04 gnorm 0.19 55,868 tok/s peakVRAM 13.3GB +[08:41:41] step 1580/4768 loss 3.3151 lr 6.00e-04 gnorm 0.17 55,867 tok/s peakVRAM 13.3GB +[08:44:48] step 1600/4768 loss 3.3048 lr 6.00e-04 gnorm 0.19 55,866 tok/s peakVRAM 13.3GB +[08:47:56] step 1620/4768 loss 3.3046 lr 6.00e-04 gnorm 0.16 55,864 tok/s peakVRAM 13.3GB +[08:51:04] step 1640/4768 loss 3.3010 lr 6.00e-04 gnorm 0.17 55,858 tok/s peakVRAM 13.3GB +[08:54:12] step 1660/4768 loss 3.3020 lr 6.00e-04 gnorm 0.18 55,856 tok/s peakVRAM 13.3GB +[08:57:19] step 1680/4768 loss 3.2901 lr 6.00e-04 gnorm 0.18 55,858 tok/s peakVRAM 13.3GB +[09:00:27] step 1700/4768 loss 3.2835 lr 6.00e-04 gnorm 0.17 55,858 tok/s peakVRAM 13.3GB +[09:03:35] step 1720/4768 loss 3.2856 lr 6.00e-04 gnorm 0.18 55,861 tok/s peakVRAM 13.3GB +[09:06:42] step 1740/4768 loss 3.2787 lr 6.00e-04 gnorm 0.16 55,859 tok/s peakVRAM 13.3GB +[09:09:50] step 1760/4768 loss 3.2778 lr 6.00e-04 gnorm 0.16 55,859 tok/s peakVRAM 13.3GB +[09:12:58] step 1780/4768 loss 3.2652 lr 6.00e-04 gnorm 0.18 55,855 tok/s peakVRAM 13.3GB +[09:16:06] step 1800/4768 loss 3.2684 lr 6.00e-04 gnorm 0.15 55,850 tok/s peakVRAM 13.3GB +[09:19:13] step 1820/4768 loss 3.2644 lr 6.00e-04 gnorm 0.19 55,853 tok/s peakVRAM 13.3GB +[09:22:21] step 1840/4768 loss 3.2582 lr 6.00e-04 gnorm 0.16 55,857 tok/s peakVRAM 13.3GB +[09:25:29] step 1860/4768 loss 3.2575 lr 6.00e-04 gnorm 0.17 55,859 tok/s peakVRAM 13.3GB +[09:28:37] step 1880/4768 loss 3.2525 lr 6.00e-04 gnorm 0.15 55,861 tok/s peakVRAM 13.3GB +[09:31:44] step 1900/4768 loss 3.2469 lr 6.00e-04 gnorm 0.17 55,868 tok/s peakVRAM 13.3GB +[09:34:52] step 1920/4768 loss 3.2459 lr 6.00e-04 gnorm 0.15 55,867 tok/s peakVRAM 13.3GB +[09:38:00] step 1940/4768 loss 3.2396 lr 6.00e-04 gnorm 0.15 55,862 tok/s peakVRAM 13.3GB +[09:41:07] step 1960/4768 loss 3.2438 lr 6.00e-04 gnorm 0.18 55,862 tok/s peakVRAM 13.3GB +[09:44:15] step 1980/4768 loss 3.2376 lr 6.00e-04 gnorm 0.15 55,860 tok/s peakVRAM 13.3GB +[09:47:23] step 2000/4768 loss 3.2313 lr 6.00e-04 gnorm 0.17 55,858 tok/s peakVRAM 13.3GB +[09:47:25] ckpt @ 2000 +[09:50:33] step 2020/4768 loss 3.2308 lr 6.00e-04 gnorm 0.15 55,181 tok/s peakVRAM 13.3GB +[09:53:41] step 2040/4768 loss 3.2255 lr 6.00e-04 gnorm 0.15 55,856 tok/s peakVRAM 13.3GB +[09:56:48] step 2060/4768 loss 3.2255 lr 6.00e-04 gnorm 0.16 55,854 tok/s peakVRAM 13.3GB +[09:59:56] step 2080/4768 loss 3.2140 lr 6.00e-04 gnorm 0.16 55,850 tok/s peakVRAM 13.3GB +[10:03:04] step 2100/4768 loss 3.2224 lr 6.00e-04 gnorm 0.15 55,847 tok/s peakVRAM 13.3GB +[10:06:12] step 2120/4768 loss 3.2110 lr 6.00e-04 gnorm 0.14 55,850 tok/s peakVRAM 13.3GB +[10:09:19] step 2140/4768 loss 3.2240 lr 6.00e-04 gnorm 0.15 55,845 tok/s peakVRAM 13.3GB +[10:12:27] step 2160/4768 loss 3.2026 lr 6.00e-04 gnorm 0.14 55,842 tok/s peakVRAM 13.3GB +[10:15:35] step 2180/4768 loss 3.2045 lr 6.00e-04 gnorm 0.16 55,843 tok/s peakVRAM 13.3GB +[10:18:43] step 2200/4768 loss 3.2058 lr 6.00e-04 gnorm 0.15 55,847 tok/s peakVRAM 13.3GB +[10:21:50] step 2220/4768 loss 3.1977 lr 6.00e-04 gnorm 0.14 55,847 tok/s peakVRAM 13.3GB +[10:24:58] step 2240/4768 loss 3.1945 lr 6.00e-04 gnorm 0.15 55,852 tok/s peakVRAM 13.3GB +[10:28:06] step 2260/4768 loss 3.1916 lr 6.00e-04 gnorm 0.14 55,853 tok/s peakVRAM 13.3GB +[10:31:14] step 2280/4768 loss 3.1868 lr 6.00e-04 gnorm 0.16 55,856 tok/s peakVRAM 13.3GB +[10:34:21] step 2300/4768 loss 3.1902 lr 6.00e-04 gnorm 0.14 55,852 tok/s peakVRAM 13.3GB +[10:37:29] step 2320/4768 loss 3.1867 lr 6.00e-04 gnorm 0.13 55,852 tok/s peakVRAM 13.3GB +[10:40:37] step 2340/4768 loss 3.1821 lr 6.00e-04 gnorm 0.14 55,849 tok/s peakVRAM 13.3GB +[10:43:45] step 2360/4768 loss 3.1724 lr 6.00e-04 gnorm 0.15 55,851 tok/s peakVRAM 13.3GB +[10:46:52] step 2380/4768 loss 3.1711 lr 6.00e-04 gnorm 0.12 55,851 tok/s peakVRAM 13.3GB +[10:50:00] step 2400/4768 loss 3.1715 lr 6.00e-04 gnorm 0.12 55,858 tok/s peakVRAM 13.3GB +[10:53:08] step 2420/4768 loss 3.1738 lr 6.00e-04 gnorm 0.14 55,855 tok/s peakVRAM 13.3GB +[10:56:16] step 2440/4768 loss 3.1681 lr 6.00e-04 gnorm 0.13 55,852 tok/s peakVRAM 13.3GB +[10:59:23] step 2460/4768 loss 3.1676 lr 6.00e-04 gnorm 0.17 55,857 tok/s peakVRAM 13.3GB +[11:02:31] step 2480/4768 loss 3.1639 lr 6.00e-04 gnorm 0.15 55,854 tok/s peakVRAM 13.3GB +[11:05:39] step 2500/4768 loss 3.1592 lr 6.00e-04 gnorm 0.17 55,856 tok/s peakVRAM 13.3GB +[11:05:41] ckpt @ 2500 +[11:08:49] step 2520/4768 loss 3.1604 lr 6.00e-04 gnorm 0.14 55,177 tok/s peakVRAM 13.3GB +[11:11:56] step 2540/4768 loss 3.1612 lr 6.00e-04 gnorm 0.15 55,851 tok/s peakVRAM 13.3GB +[11:15:04] step 2560/4768 loss 3.1578 lr 6.00e-04 gnorm 0.13 55,854 tok/s peakVRAM 13.3GB +[11:18:12] step 2580/4768 loss 3.1543 lr 6.00e-04 gnorm 0.13 55,856 tok/s peakVRAM 13.3GB +[11:21:20] step 2600/4768 loss 3.1444 lr 6.00e-04 gnorm 0.14 55,855 tok/s peakVRAM 13.3GB +[11:24:27] step 2620/4768 loss 3.1531 lr 6.00e-04 gnorm 0.13 55,850 tok/s peakVRAM 13.3GB +[11:27:35] step 2640/4768 loss 3.1535 lr 6.00e-04 gnorm 0.13 55,851 tok/s peakVRAM 13.3GB +[11:30:43] step 2660/4768 loss 3.1523 lr 6.00e-04 gnorm 0.13 55,851 tok/s peakVRAM 13.3GB +[11:33:51] step 2680/4768 loss 3.1478 lr 6.00e-04 gnorm 0.15 55,851 tok/s peakVRAM 13.3GB +[11:36:58] step 2700/4768 loss 3.1495 lr 6.00e-04 gnorm 0.13 55,849 tok/s peakVRAM 13.3GB +[11:40:06] step 2720/4768 loss 3.1323 lr 6.00e-04 gnorm 0.13 55,855 tok/s peakVRAM 13.3GB +[11:43:14] step 2740/4768 loss 3.1344 lr 6.00e-04 gnorm 0.11 55,857 tok/s peakVRAM 13.3GB +[11:46:22] step 2760/4768 loss 3.1370 lr 6.00e-04 gnorm 0.13 55,860 tok/s peakVRAM 13.3GB +[11:49:29] step 2780/4768 loss 3.1280 lr 6.00e-04 gnorm 0.15 55,860 tok/s peakVRAM 13.3GB +[11:52:37] step 2800/4768 loss 3.1230 lr 6.00e-04 gnorm 0.12 55,865 tok/s peakVRAM 13.3GB +[11:55:45] step 2820/4768 loss 3.1200 lr 6.00e-04 gnorm 0.13 55,863 tok/s peakVRAM 13.3GB +[11:58:52] step 2840/4768 loss 3.1284 lr 6.00e-04 gnorm 0.13 55,856 tok/s peakVRAM 13.3GB +[12:02:00] step 2860/4768 loss 3.1202 lr 6.00e-04 gnorm 0.13 55,854 tok/s peakVRAM 13.3GB +[12:05:08] step 2880/4768 loss 3.1225 lr 6.00e-04 gnorm 0.14 55,855 tok/s peakVRAM 13.3GB +[12:08:16] step 2900/4768 loss 3.1232 lr 6.00e-04 gnorm 0.12 55,853 tok/s peakVRAM 13.3GB +[12:11:23] step 2920/4768 loss 3.1308 lr 6.00e-04 gnorm 0.12 55,858 tok/s peakVRAM 13.3GB diff --git a/GQA/run/train.out b/GQA/run/train.out new file mode 100644 index 0000000000000000000000000000000000000000..b762f3c7db2395d7244b16fb664e543f639195ba --- /dev/null +++ b/GQA/run/train.out @@ -0,0 +1,159 @@ +[04:33:11] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[04:33:11] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[04:33:13] model GPT-ref: 178.1M param (L18 d896 h14 kv2) +[04:33:14] optimizer=muon muon_lr=0.02 (mult=33.3x) +[04:33:14] r11 pamiec: act_ckpt=True loss_chunk=1024 +[04:33:14] r13: mixer_layout=AAAAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[04:33:14] torch.compile enabled +[04:33:14] START gpt-ref: steps=4768 (od 0) lr=0.0006->6e-05 +[04:37:36] step 20/4768 loss 10.3830 lr 2.40e-05 gnorm 2.93 40,038 tok/s peakVRAM 13.3GB +[04:40:44] step 40/4768 loss 9.5665 lr 4.80e-05 gnorm 1.43 55,950 tok/s peakVRAM 13.3GB +[04:43:51] step 60/4768 loss 8.6882 lr 7.20e-05 gnorm 0.83 55,956 tok/s peakVRAM 13.3GB +[04:46:58] step 80/4768 loss 8.0380 lr 9.60e-05 gnorm 1.30 55,950 tok/s peakVRAM 13.3GB +[04:50:06] step 100/4768 loss 7.5906 lr 1.20e-04 gnorm 1.02 55,935 tok/s peakVRAM 13.3GB +[04:53:13] step 120/4768 loss 7.0952 lr 1.44e-04 gnorm 0.93 55,917 tok/s peakVRAM 13.3GB +[04:56:21] step 140/4768 loss 6.6652 lr 1.68e-04 gnorm 0.70 55,910 tok/s peakVRAM 13.3GB +[04:59:28] step 160/4768 loss 6.3258 lr 1.92e-04 gnorm 0.73 55,912 tok/s peakVRAM 13.3GB +[05:02:36] step 180/4768 loss 6.0885 lr 2.16e-04 gnorm 0.78 55,908 tok/s peakVRAM 13.3GB +[05:05:44] step 200/4768 loss 5.8922 lr 2.40e-04 gnorm 0.68 55,901 tok/s peakVRAM 13.3GB +[05:08:51] step 220/4768 loss 5.7195 lr 2.64e-04 gnorm 0.78 55,896 tok/s peakVRAM 13.3GB +[05:11:59] step 240/4768 loss 5.5645 lr 2.88e-04 gnorm 0.84 55,897 tok/s peakVRAM 13.3GB +[05:15:06] step 260/4768 loss 5.4111 lr 3.12e-04 gnorm 0.72 55,891 tok/s peakVRAM 13.3GB +[05:18:14] step 280/4768 loss 5.2646 lr 3.36e-04 gnorm 0.95 55,890 tok/s peakVRAM 13.3GB +[05:21:22] step 300/4768 loss 5.1251 lr 3.60e-04 gnorm 0.78 55,890 tok/s peakVRAM 13.3GB +[05:24:29] step 320/4768 loss 4.9982 lr 3.84e-04 gnorm 0.77 55,889 tok/s peakVRAM 13.3GB +[05:27:37] step 340/4768 loss 4.8751 lr 4.08e-04 gnorm 0.92 55,887 tok/s peakVRAM 13.3GB +[05:30:44] step 360/4768 loss 4.7710 lr 4.32e-04 gnorm 0.59 55,884 tok/s peakVRAM 13.3GB +[05:33:52] step 380/4768 loss 4.6553 lr 4.56e-04 gnorm 0.80 55,885 tok/s peakVRAM 13.3GB +[05:37:00] step 400/4768 loss 4.5465 lr 4.80e-04 gnorm 0.86 55,881 tok/s peakVRAM 13.3GB +[05:40:07] step 420/4768 loss 4.4487 lr 5.04e-04 gnorm 0.73 55,881 tok/s peakVRAM 13.3GB +[05:43:15] step 440/4768 loss 4.3597 lr 5.28e-04 gnorm 0.71 55,881 tok/s peakVRAM 13.3GB +[05:46:23] step 460/4768 loss 4.2750 lr 5.52e-04 gnorm 0.74 55,884 tok/s peakVRAM 13.3GB +[05:49:30] step 480/4768 loss 4.2145 lr 5.76e-04 gnorm 0.55 55,873 tok/s peakVRAM 13.3GB +[05:52:38] step 500/4768 loss 4.1491 lr 6.00e-04 gnorm 0.72 55,877 tok/s peakVRAM 13.3GB +[05:52:40] ckpt @ 500 +[05:55:47] step 520/4768 loss 4.1072 lr 6.00e-04 gnorm 0.50 55,403 tok/s peakVRAM 13.3GB +[05:58:55] step 540/4768 loss 4.0526 lr 6.00e-04 gnorm 0.56 55,892 tok/s peakVRAM 13.3GB +[06:02:03] step 560/4768 loss 3.9963 lr 6.00e-04 gnorm 0.45 55,879 tok/s peakVRAM 13.3GB +[06:05:10] step 580/4768 loss 3.9511 lr 6.00e-04 gnorm 0.48 55,870 tok/s peakVRAM 13.3GB +[06:08:18] step 600/4768 loss 3.9064 lr 6.00e-04 gnorm 0.47 55,866 tok/s peakVRAM 13.3GB +[06:11:26] step 620/4768 loss 3.8815 lr 6.00e-04 gnorm 0.41 55,861 tok/s peakVRAM 13.3GB +[06:14:33] step 640/4768 loss 3.8533 lr 6.00e-04 gnorm 0.38 55,857 tok/s peakVRAM 13.3GB +[06:17:41] step 660/4768 loss 3.8212 lr 6.00e-04 gnorm 0.31 55,854 tok/s peakVRAM 13.3GB +[06:20:49] step 680/4768 loss 3.7955 lr 6.00e-04 gnorm 0.35 55,851 tok/s peakVRAM 13.3GB +[06:23:57] step 700/4768 loss 3.7635 lr 6.00e-04 gnorm 0.50 55,865 tok/s peakVRAM 13.3GB +[06:27:04] step 720/4768 loss 3.7311 lr 6.00e-04 gnorm 0.39 55,892 tok/s peakVRAM 13.3GB +[06:30:12] step 740/4768 loss 3.7248 lr 6.00e-04 gnorm 0.31 55,853 tok/s peakVRAM 13.3GB +[06:33:20] step 760/4768 loss 3.6999 lr 6.00e-04 gnorm 0.39 55,855 tok/s peakVRAM 13.3GB +[06:36:27] step 780/4768 loss 3.6731 lr 6.00e-04 gnorm 0.32 55,853 tok/s peakVRAM 13.3GB +[06:39:35] step 800/4768 loss 3.6588 lr 6.00e-04 gnorm 0.30 55,854 tok/s peakVRAM 13.3GB +[06:42:43] step 820/4768 loss 3.6434 lr 6.00e-04 gnorm 0.33 55,852 tok/s peakVRAM 13.3GB +[06:45:51] step 840/4768 loss 3.6271 lr 6.00e-04 gnorm 0.34 55,855 tok/s peakVRAM 13.3GB +[06:48:58] step 860/4768 loss 3.6045 lr 6.00e-04 gnorm 0.26 55,855 tok/s peakVRAM 13.3GB +[06:52:06] step 880/4768 loss 3.5985 lr 6.00e-04 gnorm 0.24 55,857 tok/s peakVRAM 13.3GB +[06:55:14] step 900/4768 loss 3.5804 lr 6.00e-04 gnorm 0.36 55,853 tok/s peakVRAM 13.3GB +[06:58:22] step 920/4768 loss 3.5707 lr 6.00e-04 gnorm 0.28 55,850 tok/s peakVRAM 13.3GB +[07:01:29] step 940/4768 loss 3.5650 lr 6.00e-04 gnorm 0.26 55,852 tok/s peakVRAM 13.3GB +[07:04:37] step 960/4768 loss 3.5389 lr 6.00e-04 gnorm 0.27 55,853 tok/s peakVRAM 13.3GB +[07:07:45] step 980/4768 loss 3.5353 lr 6.00e-04 gnorm 0.22 55,854 tok/s peakVRAM 13.3GB +[07:10:52] step 1000/4768 loss 3.5233 lr 6.00e-04 gnorm 0.27 55,852 tok/s peakVRAM 13.3GB +[07:10:55] ckpt @ 1000 +[07:14:02] step 1020/4768 loss 3.5180 lr 6.00e-04 gnorm 0.25 55,193 tok/s peakVRAM 13.3GB +[07:17:10] step 1040/4768 loss 3.4907 lr 6.00e-04 gnorm 0.24 55,861 tok/s peakVRAM 13.3GB +[07:20:18] step 1060/4768 loss 3.5013 lr 6.00e-04 gnorm 0.23 55,863 tok/s peakVRAM 13.3GB +[07:23:26] step 1080/4768 loss 3.4905 lr 6.00e-04 gnorm 0.31 55,864 tok/s peakVRAM 13.3GB +[07:26:33] step 1100/4768 loss 3.4747 lr 6.00e-04 gnorm 0.25 55,862 tok/s peakVRAM 13.3GB +[07:29:41] step 1120/4768 loss 3.4564 lr 6.00e-04 gnorm 0.22 55,860 tok/s peakVRAM 13.3GB +[07:32:49] step 1140/4768 loss 3.4499 lr 6.00e-04 gnorm 0.26 55,860 tok/s peakVRAM 13.3GB +[07:35:56] step 1160/4768 loss 3.4432 lr 6.00e-04 gnorm 0.21 55,860 tok/s peakVRAM 13.3GB +[07:39:04] step 1180/4768 loss 3.4449 lr 6.00e-04 gnorm 0.23 55,866 tok/s peakVRAM 13.3GB +[07:42:12] step 1200/4768 loss 3.4256 lr 6.00e-04 gnorm 0.21 55,870 tok/s peakVRAM 13.3GB +[07:45:19] step 1220/4768 loss 3.4232 lr 6.00e-04 gnorm 0.24 55,868 tok/s peakVRAM 13.3GB +[07:48:27] step 1240/4768 loss 3.4146 lr 6.00e-04 gnorm 0.21 55,859 tok/s peakVRAM 13.3GB +[07:51:35] step 1260/4768 loss 3.3998 lr 6.00e-04 gnorm 0.21 55,863 tok/s peakVRAM 13.3GB +[07:54:43] step 1280/4768 loss 3.4023 lr 6.00e-04 gnorm 0.18 55,865 tok/s peakVRAM 13.3GB +[07:57:50] step 1300/4768 loss 3.3961 lr 6.00e-04 gnorm 0.25 55,864 tok/s peakVRAM 13.3GB +[08:00:58] step 1320/4768 loss 3.3871 lr 6.00e-04 gnorm 0.20 55,858 tok/s peakVRAM 13.3GB +[08:04:06] step 1340/4768 loss 3.3869 lr 6.00e-04 gnorm 0.21 55,852 tok/s peakVRAM 13.3GB +[08:07:14] step 1360/4768 loss 3.3778 lr 6.00e-04 gnorm 0.21 55,852 tok/s peakVRAM 13.3GB +[08:10:21] step 1380/4768 loss 3.3692 lr 6.00e-04 gnorm 0.20 55,856 tok/s peakVRAM 13.3GB +[08:13:29] step 1400/4768 loss 3.3585 lr 6.00e-04 gnorm 0.18 55,859 tok/s peakVRAM 13.3GB +[08:16:37] step 1420/4768 loss 3.3467 lr 6.00e-04 gnorm 0.21 55,856 tok/s peakVRAM 13.3GB +[08:19:44] step 1440/4768 loss 3.3529 lr 6.00e-04 gnorm 0.19 55,861 tok/s peakVRAM 13.3GB +[08:22:52] step 1460/4768 loss 3.3505 lr 6.00e-04 gnorm 0.18 55,860 tok/s peakVRAM 13.3GB +[08:26:00] step 1480/4768 loss 3.3544 lr 6.00e-04 gnorm 0.22 55,860 tok/s peakVRAM 13.3GB +[08:29:08] step 1500/4768 loss 3.3271 lr 6.00e-04 gnorm 0.18 55,857 tok/s peakVRAM 13.3GB +[08:29:10] ckpt @ 1500 +[08:32:18] step 1520/4768 loss 3.3344 lr 6.00e-04 gnorm 0.19 55,160 tok/s peakVRAM 13.3GB +[08:35:25] step 1540/4768 loss 3.3192 lr 6.00e-04 gnorm 0.23 55,858 tok/s peakVRAM 13.3GB +[08:38:33] step 1560/4768 loss 3.3266 lr 6.00e-04 gnorm 0.19 55,868 tok/s peakVRAM 13.3GB +[08:41:41] step 1580/4768 loss 3.3151 lr 6.00e-04 gnorm 0.17 55,867 tok/s peakVRAM 13.3GB +[08:44:48] step 1600/4768 loss 3.3048 lr 6.00e-04 gnorm 0.19 55,866 tok/s peakVRAM 13.3GB +[08:47:56] step 1620/4768 loss 3.3046 lr 6.00e-04 gnorm 0.16 55,864 tok/s peakVRAM 13.3GB +[08:51:04] step 1640/4768 loss 3.3010 lr 6.00e-04 gnorm 0.17 55,858 tok/s peakVRAM 13.3GB +[08:54:12] step 1660/4768 loss 3.3020 lr 6.00e-04 gnorm 0.18 55,856 tok/s peakVRAM 13.3GB +[08:57:19] step 1680/4768 loss 3.2901 lr 6.00e-04 gnorm 0.18 55,858 tok/s peakVRAM 13.3GB +[09:00:27] step 1700/4768 loss 3.2835 lr 6.00e-04 gnorm 0.17 55,858 tok/s peakVRAM 13.3GB +[09:03:35] step 1720/4768 loss 3.2856 lr 6.00e-04 gnorm 0.18 55,861 tok/s peakVRAM 13.3GB +[09:06:42] step 1740/4768 loss 3.2787 lr 6.00e-04 gnorm 0.16 55,859 tok/s peakVRAM 13.3GB +[09:09:50] step 1760/4768 loss 3.2778 lr 6.00e-04 gnorm 0.16 55,859 tok/s peakVRAM 13.3GB +[09:12:58] step 1780/4768 loss 3.2652 lr 6.00e-04 gnorm 0.18 55,855 tok/s peakVRAM 13.3GB +[09:16:06] step 1800/4768 loss 3.2684 lr 6.00e-04 gnorm 0.15 55,850 tok/s peakVRAM 13.3GB +[09:19:13] step 1820/4768 loss 3.2644 lr 6.00e-04 gnorm 0.19 55,853 tok/s peakVRAM 13.3GB +[09:22:21] step 1840/4768 loss 3.2582 lr 6.00e-04 gnorm 0.16 55,857 tok/s peakVRAM 13.3GB +[09:25:29] step 1860/4768 loss 3.2575 lr 6.00e-04 gnorm 0.17 55,859 tok/s peakVRAM 13.3GB +[09:28:37] step 1880/4768 loss 3.2525 lr 6.00e-04 gnorm 0.15 55,861 tok/s peakVRAM 13.3GB +[09:31:44] step 1900/4768 loss 3.2469 lr 6.00e-04 gnorm 0.17 55,868 tok/s peakVRAM 13.3GB +[09:34:52] step 1920/4768 loss 3.2459 lr 6.00e-04 gnorm 0.15 55,867 tok/s peakVRAM 13.3GB +[09:38:00] step 1940/4768 loss 3.2396 lr 6.00e-04 gnorm 0.15 55,862 tok/s peakVRAM 13.3GB +[09:41:07] step 1960/4768 loss 3.2438 lr 6.00e-04 gnorm 0.18 55,862 tok/s peakVRAM 13.3GB +[09:44:15] step 1980/4768 loss 3.2376 lr 6.00e-04 gnorm 0.15 55,860 tok/s peakVRAM 13.3GB +[09:47:23] step 2000/4768 loss 3.2313 lr 6.00e-04 gnorm 0.17 55,858 tok/s peakVRAM 13.3GB +[09:47:25] ckpt @ 2000 +[09:50:33] step 2020/4768 loss 3.2308 lr 6.00e-04 gnorm 0.15 55,181 tok/s peakVRAM 13.3GB +[09:53:41] step 2040/4768 loss 3.2255 lr 6.00e-04 gnorm 0.15 55,856 tok/s peakVRAM 13.3GB +[09:56:48] step 2060/4768 loss 3.2255 lr 6.00e-04 gnorm 0.16 55,854 tok/s peakVRAM 13.3GB +[09:59:56] step 2080/4768 loss 3.2140 lr 6.00e-04 gnorm 0.16 55,850 tok/s peakVRAM 13.3GB +[10:03:04] step 2100/4768 loss 3.2224 lr 6.00e-04 gnorm 0.15 55,847 tok/s peakVRAM 13.3GB +[10:06:12] step 2120/4768 loss 3.2110 lr 6.00e-04 gnorm 0.14 55,850 tok/s peakVRAM 13.3GB +[10:09:19] step 2140/4768 loss 3.2240 lr 6.00e-04 gnorm 0.15 55,845 tok/s peakVRAM 13.3GB +[10:12:27] step 2160/4768 loss 3.2026 lr 6.00e-04 gnorm 0.14 55,842 tok/s peakVRAM 13.3GB +[10:15:35] step 2180/4768 loss 3.2045 lr 6.00e-04 gnorm 0.16 55,843 tok/s peakVRAM 13.3GB +[10:18:43] step 2200/4768 loss 3.2058 lr 6.00e-04 gnorm 0.15 55,847 tok/s peakVRAM 13.3GB +[10:21:50] step 2220/4768 loss 3.1977 lr 6.00e-04 gnorm 0.14 55,847 tok/s peakVRAM 13.3GB +[10:24:58] step 2240/4768 loss 3.1945 lr 6.00e-04 gnorm 0.15 55,852 tok/s peakVRAM 13.3GB +[10:28:06] step 2260/4768 loss 3.1916 lr 6.00e-04 gnorm 0.14 55,853 tok/s peakVRAM 13.3GB +[10:31:14] step 2280/4768 loss 3.1868 lr 6.00e-04 gnorm 0.16 55,856 tok/s peakVRAM 13.3GB +[10:34:21] step 2300/4768 loss 3.1902 lr 6.00e-04 gnorm 0.14 55,852 tok/s peakVRAM 13.3GB +[10:37:29] step 2320/4768 loss 3.1867 lr 6.00e-04 gnorm 0.13 55,852 tok/s peakVRAM 13.3GB +[10:40:37] step 2340/4768 loss 3.1821 lr 6.00e-04 gnorm 0.14 55,849 tok/s peakVRAM 13.3GB +[10:43:45] step 2360/4768 loss 3.1724 lr 6.00e-04 gnorm 0.15 55,851 tok/s peakVRAM 13.3GB +[10:46:52] step 2380/4768 loss 3.1711 lr 6.00e-04 gnorm 0.12 55,851 tok/s peakVRAM 13.3GB +[10:50:00] step 2400/4768 loss 3.1715 lr 6.00e-04 gnorm 0.12 55,858 tok/s peakVRAM 13.3GB +[10:53:08] step 2420/4768 loss 3.1738 lr 6.00e-04 gnorm 0.14 55,855 tok/s peakVRAM 13.3GB +[10:56:16] step 2440/4768 loss 3.1681 lr 6.00e-04 gnorm 0.13 55,852 tok/s peakVRAM 13.3GB +[10:59:23] step 2460/4768 loss 3.1676 lr 6.00e-04 gnorm 0.17 55,857 tok/s peakVRAM 13.3GB +[11:02:31] step 2480/4768 loss 3.1639 lr 6.00e-04 gnorm 0.15 55,854 tok/s peakVRAM 13.3GB +[11:05:39] step 2500/4768 loss 3.1592 lr 6.00e-04 gnorm 0.17 55,856 tok/s peakVRAM 13.3GB +[11:05:41] ckpt @ 2500 +[11:08:49] step 2520/4768 loss 3.1604 lr 6.00e-04 gnorm 0.14 55,177 tok/s peakVRAM 13.3GB +[11:11:56] step 2540/4768 loss 3.1612 lr 6.00e-04 gnorm 0.15 55,851 tok/s peakVRAM 13.3GB +[11:15:04] step 2560/4768 loss 3.1578 lr 6.00e-04 gnorm 0.13 55,854 tok/s peakVRAM 13.3GB +[11:18:12] step 2580/4768 loss 3.1543 lr 6.00e-04 gnorm 0.13 55,856 tok/s peakVRAM 13.3GB +[11:21:20] step 2600/4768 loss 3.1444 lr 6.00e-04 gnorm 0.14 55,855 tok/s peakVRAM 13.3GB +[11:24:27] step 2620/4768 loss 3.1531 lr 6.00e-04 gnorm 0.13 55,850 tok/s peakVRAM 13.3GB +[11:27:35] step 2640/4768 loss 3.1535 lr 6.00e-04 gnorm 0.13 55,851 tok/s peakVRAM 13.3GB +[11:30:43] step 2660/4768 loss 3.1523 lr 6.00e-04 gnorm 0.13 55,851 tok/s peakVRAM 13.3GB +[11:33:51] step 2680/4768 loss 3.1478 lr 6.00e-04 gnorm 0.15 55,851 tok/s peakVRAM 13.3GB +[11:36:58] step 2700/4768 loss 3.1495 lr 6.00e-04 gnorm 0.13 55,849 tok/s peakVRAM 13.3GB +[11:40:06] step 2720/4768 loss 3.1323 lr 6.00e-04 gnorm 0.13 55,855 tok/s peakVRAM 13.3GB +[11:43:14] step 2740/4768 loss 3.1344 lr 6.00e-04 gnorm 0.11 55,857 tok/s peakVRAM 13.3GB +[11:46:22] step 2760/4768 loss 3.1370 lr 6.00e-04 gnorm 0.13 55,860 tok/s peakVRAM 13.3GB +[11:49:29] step 2780/4768 loss 3.1280 lr 6.00e-04 gnorm 0.15 55,860 tok/s peakVRAM 13.3GB +[11:52:37] step 2800/4768 loss 3.1230 lr 6.00e-04 gnorm 0.12 55,865 tok/s peakVRAM 13.3GB +[11:55:45] step 2820/4768 loss 3.1200 lr 6.00e-04 gnorm 0.13 55,863 tok/s peakVRAM 13.3GB +[11:58:52] step 2840/4768 loss 3.1284 lr 6.00e-04 gnorm 0.13 55,856 tok/s peakVRAM 13.3GB +[12:02:00] step 2860/4768 loss 3.1202 lr 6.00e-04 gnorm 0.13 55,854 tok/s peakVRAM 13.3GB +[12:05:08] step 2880/4768 loss 3.1225 lr 6.00e-04 gnorm 0.14 55,855 tok/s peakVRAM 13.3GB +[12:08:16] step 2900/4768 loss 3.1232 lr 6.00e-04 gnorm 0.12 55,853 tok/s peakVRAM 13.3GB +[12:11:23] step 2920/4768 loss 3.1308 lr 6.00e-04 gnorm 0.12 55,858 tok/s peakVRAM 13.3GB diff --git a/K12/run/SHA256SUMS_kopiarka b/K12/run/SHA256SUMS_kopiarka new file mode 100644 index 0000000000000000000000000000000000000000..3cdbf37d9f7959889f3c2be39e215843e85122e5 --- /dev/null +++ b/K12/run/SHA256SUMS_kopiarka @@ -0,0 +1,6 @@ +846c7cc9f3c88fcda1394e757dd024eebfefc3b43cba549c4dfaa0c0efeb1594 ckpt_500.pt +84f431e41c1a7dcd3cb3a1eda8a572094350a7dbd998669bca640e036c9fd923 ckpt_1000.pt +d3bd736d8c42dbd52568c0598ad52862dbddbf69829cbf6325faa3bf0a4083cc ckpt_1500.pt +cdc444a304c2f57727937a5ca0f58599d220eb0741a3920ddf73f3965e2117ca ckpt_2000.pt +8f5298ada65ab57bfe1b63dcd49b956cf0f37f26e7ec5e2bab072e2e8da21e84 ckpt_2500.pt +c54e324a6d5e2b9e33bf9731ff4162df18fb774e57e7bab253576f096dd24511 ckpt_3000.pt diff --git a/K12/run/args.txt b/K12/run/args.txt new file mode 100644 index 0000000000000000000000000000000000000000..951fce6cb1254d74ad60d529abfcd03cab738945 --- /dev/null +++ b/K12/run/args.txt @@ -0,0 +1 @@ +--data-dir /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2 --out-dir /home/ubuntu/kolektyw/dowody400m/wyniki/K12 --run-id dowody_K12 --vocab 32768 --dtype uint16 --block 4096 --n-layer 12 --n-embd 1024 --n-head 16 --value-residual --batch 128 --grad-accum 8 --norm rmsnorm --pos rope --rope-theta 500000 --ffn swiglu --ffn-mult 2.667 --qk-norm --attn-bias 0 --optimizer muon --muon-lr 0.02 --lr 6e-4 --min-lr 6e-05 --wd 0.1 --schedule wsd --decay-frac 0.2 --warmup 500 --act-ckpt --loss-chunk 1024 --sampler random --seed 1 --data-seed 20261022 --compile --log-every 20 --eval-every 4769 --eval-iters 1 --steps 4768 --ckpt-every 500 \ No newline at end of file diff --git a/K12/run/metrics.jsonl b/K12/run/metrics.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5cfd22180c77fbbc1226e9de38d73ac543ba7356 --- /dev/null +++ b/K12/run/metrics.jsonl @@ -0,0 +1,153 @@ +{"step": 20, "loss": 10.404603719711304, "lr": 2.3999999999999997e-05, "tok_s": 43862.84977431825} +{"step": 40, "loss": 9.534286546707154, "lr": 4.7999999999999994e-05, "tok_s": 58708.75008696259} +{"step": 60, "loss": 8.592787218093871, "lr": 7.199999999999999e-05, "tok_s": 58728.55645155153} +{"step": 80, "loss": 7.972748017311096, "lr": 9.599999999999999e-05, "tok_s": 58710.56110012293} +{"step": 100, "loss": 7.537906694412231, "lr": 0.00011999999999999999, "tok_s": 58682.614600713896} +{"step": 120, "loss": 7.051306915283203, "lr": 0.00014399999999999998, "tok_s": 58663.493826476195} +{"step": 140, "loss": 6.628354525566101, "lr": 0.000168, "tok_s": 58646.106530416655} +{"step": 160, "loss": 6.276793456077575, "lr": 0.00019199999999999998, "tok_s": 58628.3912037025} +{"step": 180, "loss": 6.018369221687317, "lr": 0.00021599999999999996, "tok_s": 58626.09479047179} +{"step": 200, "loss": 5.8070940494537355, "lr": 0.00023999999999999998, "tok_s": 58611.57576779736} +{"step": 220, "loss": 5.628641867637635, "lr": 0.00026399999999999997, "tok_s": 58611.74886020378} +{"step": 240, "loss": 5.462792611122131, "lr": 0.00028799999999999995, "tok_s": 58622.5650233335} +{"step": 260, "loss": 5.3006878137588505, "lr": 0.000312, "tok_s": 58625.83103957974} +{"step": 280, "loss": 5.149684000015259, "lr": 0.000336, "tok_s": 58625.33581950116} +{"step": 300, "loss": 5.010294389724732, "lr": 0.00035999999999999997, "tok_s": 58621.081819644285} +{"step": 320, "loss": 4.881231045722961, "lr": 0.00038399999999999996, "tok_s": 58614.27217303556} +{"step": 340, "loss": 4.742927312850952, "lr": 0.000408, "tok_s": 58615.62371070732} +{"step": 360, "loss": 4.616938376426697, "lr": 0.00043199999999999993, "tok_s": 58614.1403114175} +{"step": 380, "loss": 4.496828222274781, "lr": 0.00045599999999999997, "tok_s": 58616.3796178055} +{"step": 400, "loss": 4.392631483078003, "lr": 0.00047999999999999996, "tok_s": 58608.62616789729} +{"step": 420, "loss": 4.309436774253845, "lr": 0.000504, "tok_s": 58610.82326513075} +{"step": 440, "loss": 4.235170435905457, "lr": 0.0005279999999999999, "tok_s": 58611.922500409455} +{"step": 460, "loss": 4.16366171836853, "lr": 0.000552, "tok_s": 58611.618884582764} +{"step": 480, "loss": 4.1190622568130495, "lr": 0.0005759999999999999, "tok_s": 58612.602230085446} +{"step": 500, "loss": 4.061825442314148, "lr": 0.0006, "tok_s": 58608.78963625683} +{"step": 520, "loss": 4.027750384807587, "lr": 0.0006, "tok_s": 58065.16923676389} +{"step": 540, "loss": 3.9765324115753176, "lr": 0.0006, "tok_s": 58620.23531294467} +{"step": 560, "loss": 3.9294686794281004, "lr": 0.0006, "tok_s": 58620.60480690887} +{"step": 580, "loss": 3.8874564170837402, "lr": 0.0006, "tok_s": 58616.54695746107} +{"step": 600, "loss": 3.848431408405304, "lr": 0.0006, "tok_s": 58616.59945637344} +{"step": 620, "loss": 3.8276182293891905, "lr": 0.0006, "tok_s": 58610.03383764811} +{"step": 640, "loss": 3.8016713857650757, "lr": 0.0006, "tok_s": 58612.397497159815} +{"step": 660, "loss": 3.77252653837204, "lr": 0.0006, "tok_s": 58614.99515708387} +{"step": 680, "loss": 3.7515096426010133, "lr": 0.0006, "tok_s": 58607.41678029385} +{"step": 700, "loss": 3.7216542243957518, "lr": 0.0006, "tok_s": 58603.47148561657} +{"step": 720, "loss": 3.6929709196090696, "lr": 0.0006, "tok_s": 58598.80060614448} +{"step": 740, "loss": 3.6890414118766786, "lr": 0.0006, "tok_s": 58599.98504322484} +{"step": 760, "loss": 3.664959669113159, "lr": 0.0006, "tok_s": 58595.98211951457} +{"step": 780, "loss": 3.6401920318603516, "lr": 0.0006, "tok_s": 58596.79529233795} +{"step": 800, "loss": 3.6265568852424623, "lr": 0.0006, "tok_s": 58600.643414189704} +{"step": 820, "loss": 3.611056935787201, "lr": 0.0006, "tok_s": 58598.78522517366} +{"step": 840, "loss": 3.5982728123664858, "lr": 0.0006, "tok_s": 58600.3990999932} +{"step": 860, "loss": 3.577110970020294, "lr": 0.0006, "tok_s": 58597.851139116254} +{"step": 880, "loss": 3.571992111206055, "lr": 0.0006, "tok_s": 58592.335766610355} +{"step": 900, "loss": 3.5544564604759215, "lr": 0.0006, "tok_s": 58586.29838538495} +{"step": 920, "loss": 3.547284722328186, "lr": 0.0006, "tok_s": 58593.34695760348} +{"step": 940, "loss": 3.541924238204956, "lr": 0.0006, "tok_s": 58587.187695480054} +{"step": 960, "loss": 3.517482328414917, "lr": 0.0006, "tok_s": 58588.60627705455} +{"step": 980, "loss": 3.5142526626586914, "lr": 0.0006, "tok_s": 58594.897143534705} +{"step": 1000, "loss": 3.5042595744132994, "lr": 0.0006, "tok_s": 58588.46009192064} +{"step": 1020, "loss": 3.4992708206176757, "lr": 0.0006, "tok_s": 57812.14930900983} +{"step": 1040, "loss": 3.4728254079818726, "lr": 0.0006, "tok_s": 58591.35544269182} +{"step": 1060, "loss": 3.48453768491745, "lr": 0.0006, "tok_s": 58588.209557429254} +{"step": 1080, "loss": 3.473938596248627, "lr": 0.0006, "tok_s": 58587.41902198655} +{"step": 1100, "loss": 3.4584452748298644, "lr": 0.0006, "tok_s": 58591.59749589408} +{"step": 1120, "loss": 3.440518116950989, "lr": 0.0006, "tok_s": 58594.82516712788} +{"step": 1140, "loss": 3.4338117957115175, "lr": 0.0006, "tok_s": 58588.28768337564} +{"step": 1160, "loss": 3.4283645272254946, "lr": 0.0006, "tok_s": 58584.633001453476} +{"step": 1180, "loss": 3.4317108631134032, "lr": 0.0006, "tok_s": 58589.91112681225} +{"step": 1200, "loss": 3.413036358356476, "lr": 0.0006, "tok_s": 58581.13904649471} +{"step": 1220, "loss": 3.4098459243774415, "lr": 0.0006, "tok_s": 58583.53291667873} +{"step": 1240, "loss": 3.4025972723960876, "lr": 0.0006, "tok_s": 58586.346225600704} +{"step": 1260, "loss": 3.388778018951416, "lr": 0.0006, "tok_s": 58585.91098397099} +{"step": 1280, "loss": 3.391374719142914, "lr": 0.0006, "tok_s": 58588.81107795649} +{"step": 1300, "loss": 3.3844142317771913, "lr": 0.0006, "tok_s": 58585.81171513435} +{"step": 1320, "loss": 3.376672458648682, "lr": 0.0006, "tok_s": 58590.171822747536} +{"step": 1340, "loss": 3.376765549182892, "lr": 0.0006, "tok_s": 58587.03667865518} +{"step": 1360, "loss": 3.368115746974945, "lr": 0.0006, "tok_s": 58590.08042280073} +{"step": 1380, "loss": 3.359369957447052, "lr": 0.0006, "tok_s": 58579.82163520031} +{"step": 1400, "loss": 3.3492789387702944, "lr": 0.0006, "tok_s": 58579.18729517547} +{"step": 1420, "loss": 3.337015962600708, "lr": 0.0006, "tok_s": 58575.79650089553} +{"step": 1440, "loss": 3.3437799215316772, "lr": 0.0006, "tok_s": 58579.68236010638} +{"step": 1460, "loss": 3.3416819334030152, "lr": 0.0006, "tok_s": 58573.27415937886} +{"step": 1480, "loss": 3.345796358585358, "lr": 0.0006, "tok_s": 58572.91501271516} +{"step": 1500, "loss": 3.3184831380844115, "lr": 0.0006, "tok_s": 58573.43758661024} +{"step": 1520, "loss": 3.3255487084388733, "lr": 0.0006, "tok_s": 57778.309994633084} +{"step": 1540, "loss": 3.3099217891693113, "lr": 0.0006, "tok_s": 58532.41938083946} +{"step": 1560, "loss": 3.318115973472595, "lr": 0.0006, "tok_s": 58532.337041475155} +{"step": 1580, "loss": 3.306504261493683, "lr": 0.0006, "tok_s": 58567.39411608857} +{"step": 1600, "loss": 3.296400547027588, "lr": 0.0006, "tok_s": 58572.257498665596} +{"step": 1620, "loss": 3.2961339712142945, "lr": 0.0006, "tok_s": 58576.22262042889} +{"step": 1640, "loss": 3.2931604027748107, "lr": 0.0006, "tok_s": 58574.65899026576} +{"step": 1660, "loss": 3.293904662132263, "lr": 0.0006, "tok_s": 58564.831711681014} +{"step": 1680, "loss": 3.2828875541687013, "lr": 0.0006, "tok_s": 58562.07684675201} +{"step": 1700, "loss": 3.2760611534118653, "lr": 0.0006, "tok_s": 58569.60543857275} +{"step": 1720, "loss": 3.2780789136886597, "lr": 0.0006, "tok_s": 58565.877672177565} +{"step": 1740, "loss": 3.271463096141815, "lr": 0.0006, "tok_s": 58556.22727373207} +{"step": 1760, "loss": 3.2703675389289857, "lr": 0.0006, "tok_s": 58551.24721203557} +{"step": 1780, "loss": 3.2580028653144835, "lr": 0.0006, "tok_s": 58558.9586651728} +{"step": 1800, "loss": 3.261461687088013, "lr": 0.0006, "tok_s": 58560.024688794445} +{"step": 1820, "loss": 3.257357358932495, "lr": 0.0006, "tok_s": 58560.431397326596} +{"step": 1840, "loss": 3.2504354238510134, "lr": 0.0006, "tok_s": 58555.27708088719} +{"step": 1860, "loss": 3.249496328830719, "lr": 0.0006, "tok_s": 58571.49820492442} +{"step": 1880, "loss": 3.2457431077957155, "lr": 0.0006, "tok_s": 58553.54478125217} +{"step": 1900, "loss": 3.2397770166397093, "lr": 0.0006, "tok_s": 58561.86201764043} +{"step": 1920, "loss": 3.240020728111267, "lr": 0.0006, "tok_s": 58556.22703984424} +{"step": 1940, "loss": 3.2326427340507506, "lr": 0.0006, "tok_s": 58556.623872204094} +{"step": 1960, "loss": 3.236352801322937, "lr": 0.0006, "tok_s": 58558.79929516935} +{"step": 1980, "loss": 3.2304039359092713, "lr": 0.0006, "tok_s": 58575.08602940318} +{"step": 2000, "loss": 3.224052131175995, "lr": 0.0006, "tok_s": 58571.13845602058} +{"step": 2020, "loss": 3.223966634273529, "lr": 0.0006, "tok_s": 57804.03149473105} +{"step": 2040, "loss": 3.2166189670562746, "lr": 0.0006, "tok_s": 58583.05097441445} +{"step": 2060, "loss": 3.21760436296463, "lr": 0.0006, "tok_s": 58578.82191219394} +{"step": 2080, "loss": 3.2062864303588867, "lr": 0.0006, "tok_s": 58579.01416096729} +{"step": 2100, "loss": 3.214371931552887, "lr": 0.0006, "tok_s": 58578.60610196551} +{"step": 2120, "loss": 3.2036935448646546, "lr": 0.0006, "tok_s": 58579.446569272186} +{"step": 2140, "loss": 3.217024397850037, "lr": 0.0006, "tok_s": 58588.06594983953} +{"step": 2160, "loss": 3.1952428817749023, "lr": 0.0006, "tok_s": 58584.55636797676} +{"step": 2180, "loss": 3.1975870609283445, "lr": 0.0006, "tok_s": 58580.69701578473} +{"step": 2200, "loss": 3.198507809638977, "lr": 0.0006, "tok_s": 58583.069390456294} +{"step": 2220, "loss": 3.1905167579650877, "lr": 0.0006, "tok_s": 58578.568417262235} +{"step": 2240, "loss": 3.18764910697937, "lr": 0.0006, "tok_s": 58565.70890596694} +{"step": 2260, "loss": 3.18397171497345, "lr": 0.0006, "tok_s": 58565.45965783414} +{"step": 2280, "loss": 3.1804327368736267, "lr": 0.0006, "tok_s": 58564.653437321234} +{"step": 2300, "loss": 3.1841544985771177, "lr": 0.0006, "tok_s": 58561.42214923009} +{"step": 2320, "loss": 3.1800976395606995, "lr": 0.0006, "tok_s": 58572.766487667184} +{"step": 2340, "loss": 3.1748282670974732, "lr": 0.0006, "tok_s": 58584.013462213494} +{"step": 2360, "loss": 3.16541702747345, "lr": 0.0006, "tok_s": 58581.646707292864} +{"step": 2380, "loss": 3.1643529653549196, "lr": 0.0006, "tok_s": 58584.37235480553} +{"step": 2400, "loss": 3.1648273706436156, "lr": 0.0006, "tok_s": 58586.16064033874} +{"step": 2420, "loss": 3.1675984501838683, "lr": 0.0006, "tok_s": 58585.525850858976} +{"step": 2440, "loss": 3.1628398299217224, "lr": 0.0006, "tok_s": 58580.35244713431} +{"step": 2460, "loss": 3.1619457602500916, "lr": 0.0006, "tok_s": 58582.353124208545} +{"step": 2480, "loss": 3.1585049986839295, "lr": 0.0006, "tok_s": 58588.8177902054} +{"step": 2500, "loss": 3.1526406526565554, "lr": 0.0006, "tok_s": 58584.57579946054} +{"step": 2520, "loss": 3.154686999320984, "lr": 0.0006, "tok_s": 57829.54215959935} +{"step": 2540, "loss": 3.1545638918876646, "lr": 0.0006, "tok_s": 58616.96812325964} +{"step": 2560, "loss": 3.151969051361084, "lr": 0.0006, "tok_s": 58598.46300740269} +{"step": 2580, "loss": 3.146875965595245, "lr": 0.0006, "tok_s": 58572.50220231211} +{"step": 2600, "loss": 3.1376704573631287, "lr": 0.0006, "tok_s": 58616.03166062821} +{"step": 2620, "loss": 3.1459038615226746, "lr": 0.0006, "tok_s": 58586.39359763685} +{"step": 2640, "loss": 3.1474135518074036, "lr": 0.0006, "tok_s": 58584.45702569282} +{"step": 2660, "loss": 3.1453431606292725, "lr": 0.0006, "tok_s": 58588.917693626856} +{"step": 2680, "loss": 3.140736925601959, "lr": 0.0006, "tok_s": 58578.20811337535} +{"step": 2700, "loss": 3.143047571182251, "lr": 0.0006, "tok_s": 58575.02837797594} +{"step": 2720, "loss": 3.1259115099906922, "lr": 0.0006, "tok_s": 58573.31909202556} +{"step": 2740, "loss": 3.127363657951355, "lr": 0.0006, "tok_s": 58563.766058918976} +{"step": 2760, "loss": 3.130553150177002, "lr": 0.0006, "tok_s": 58568.07078550068} +{"step": 2780, "loss": 3.1217617988586426, "lr": 0.0006, "tok_s": 58567.39442805796} +{"step": 2800, "loss": 3.1166795253753663, "lr": 0.0006, "tok_s": 58559.65580217986} +{"step": 2820, "loss": 3.1136237382888794, "lr": 0.0006, "tok_s": 58563.632006992724} +{"step": 2840, "loss": 3.122041928768158, "lr": 0.0006, "tok_s": 58584.55675816707} +{"step": 2860, "loss": 3.1131366968154905, "lr": 0.0006, "tok_s": 58601.91811302135} +{"step": 2880, "loss": 3.1153586983680723, "lr": 0.0006, "tok_s": 58558.33163810221} +{"step": 2900, "loss": 3.116631364822388, "lr": 0.0006, "tok_s": 58604.634323974504} +{"step": 2920, "loss": 3.123911237716675, "lr": 0.0006, "tok_s": 58617.125700667} +{"step": 2940, "loss": 3.1143979907035826, "lr": 0.0006, "tok_s": 58618.255485958936} +{"step": 2960, "loss": 3.116018295288086, "lr": 0.0006, "tok_s": 58609.45960832013} +{"step": 2980, "loss": 3.1031396865844725, "lr": 0.0006, "tok_s": 58601.60757331772} +{"step": 3000, "loss": 3.1038497567176817, "lr": 0.0006, "tok_s": 58569.51808067913} +{"step": 3020, "loss": 3.099448227882385, "lr": 0.0006, "tok_s": 57789.27866566382} +{"step": 3040, "loss": 3.09664865732193, "lr": 0.0006, "tok_s": 58561.89679567643} +{"step": 3060, "loss": 3.100166177749634, "lr": 0.0006, "tok_s": 58560.26196074277} diff --git a/K12/run/srodowisko_K12_mini0_resume0.txt b/K12/run/srodowisko_K12_mini0_resume0.txt new file mode 100644 index 0000000000000000000000000000000000000000..efd308e1f97cc92352554c603af5e1aff8f012ae --- /dev/null +++ b/K12/run/srodowisko_K12_mini0_resume0.txt @@ -0,0 +1,64 @@ +data 2026-10-23T02:33:05Z ARM K12 GPU 1 MINI 0 DATA_SEED 20261022 CKPT 500 RESUME 0 DEST /var/lib/docker/kolektyw/dowody400m_ckpt/wyniki_K12 +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +0702c32c0ad602d8d12b16539e05cfbc68d1823cfd39b82b984357d78f27f664 /home/ubuntu/kolektyw/dowody400m/kod/ramie_v03.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +341aae81ce8512294dcd7de93ced26053025677b4ab449606be989f7fd898d27 /home/ubuntu/kolektyw/dowody400m/kod/kopiarka_ckpt.sh +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 diff --git a/K12/run/srodowisko_resume0.txt b/K12/run/srodowisko_resume0.txt new file mode 100644 index 0000000000000000000000000000000000000000..efd308e1f97cc92352554c603af5e1aff8f012ae --- /dev/null +++ b/K12/run/srodowisko_resume0.txt @@ -0,0 +1,64 @@ +data 2026-10-23T02:33:05Z ARM K12 GPU 1 MINI 0 DATA_SEED 20261022 CKPT 500 RESUME 0 DEST /var/lib/docker/kolektyw/dowody400m_ckpt/wyniki_K12 +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +0702c32c0ad602d8d12b16539e05cfbc68d1823cfd39b82b984357d78f27f664 /home/ubuntu/kolektyw/dowody400m/kod/ramie_v03.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +341aae81ce8512294dcd7de93ced26053025677b4ab449606be989f7fd898d27 /home/ubuntu/kolektyw/dowody400m/kod/kopiarka_ckpt.sh +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 diff --git a/K12/run/train.log b/K12/run/train.log new file mode 100644 index 0000000000000000000000000000000000000000..fcf3b585978edecee6f731add0a45426c0355754 --- /dev/null +++ b/K12/run/train.log @@ -0,0 +1,167 @@ +[04:33:11] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[04:33:11] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[04:33:13] model GPT-ref: 184.6M param (L12 d1024 h16) +[04:33:14] optimizer=muon muon_lr=0.02 (mult=33.3x) +[04:33:14] r11 pamiec: act_ckpt=True loss_chunk=1024 +[04:33:14] r13: mixer_layout=AAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[04:33:15] torch.compile enabled +[04:33:15] START gpt-ref: steps=4768 (od 0) lr=0.0006->6e-05 +[04:37:14] step 20/4768 loss 10.4046 lr 2.40e-05 gnorm 2.87 43,863 tok/s peakVRAM 14.2GB +[04:40:12] step 40/4768 loss 9.5343 lr 4.80e-05 gnorm 1.38 58,709 tok/s peakVRAM 14.2GB +[04:43:11] step 60/4768 loss 8.5928 lr 7.20e-05 gnorm 0.59 58,729 tok/s peakVRAM 14.2GB +[04:46:09] step 80/4768 loss 7.9727 lr 9.60e-05 gnorm 0.86 58,711 tok/s peakVRAM 14.2GB +[04:49:08] step 100/4768 loss 7.5379 lr 1.20e-04 gnorm 0.89 58,683 tok/s peakVRAM 14.2GB +[04:52:07] step 120/4768 loss 7.0513 lr 1.44e-04 gnorm 0.87 58,663 tok/s peakVRAM 14.2GB +[04:55:06] step 140/4768 loss 6.6284 lr 1.68e-04 gnorm 0.71 58,646 tok/s peakVRAM 14.2GB +[04:58:05] step 160/4768 loss 6.2768 lr 1.92e-04 gnorm 0.77 58,628 tok/s peakVRAM 14.2GB +[05:01:03] step 180/4768 loss 6.0184 lr 2.16e-04 gnorm 0.77 58,626 tok/s peakVRAM 14.2GB +[05:04:02] step 200/4768 loss 5.8071 lr 2.40e-04 gnorm 0.83 58,612 tok/s peakVRAM 14.2GB +[05:07:01] step 220/4768 loss 5.6286 lr 2.64e-04 gnorm 0.80 58,612 tok/s peakVRAM 14.2GB +[05:10:00] step 240/4768 loss 5.4628 lr 2.88e-04 gnorm 0.86 58,623 tok/s peakVRAM 14.2GB +[05:12:59] step 260/4768 loss 5.3007 lr 3.12e-04 gnorm 0.86 58,626 tok/s peakVRAM 14.2GB +[05:15:58] step 280/4768 loss 5.1497 lr 3.36e-04 gnorm 0.74 58,625 tok/s peakVRAM 14.2GB +[05:18:57] step 300/4768 loss 5.0103 lr 3.60e-04 gnorm 0.83 58,621 tok/s peakVRAM 14.2GB +[05:21:56] step 320/4768 loss 4.8812 lr 3.84e-04 gnorm 0.59 58,614 tok/s peakVRAM 14.2GB +[05:24:54] step 340/4768 loss 4.7429 lr 4.08e-04 gnorm 0.76 58,616 tok/s peakVRAM 14.2GB +[05:27:53] step 360/4768 loss 4.6169 lr 4.32e-04 gnorm 0.55 58,614 tok/s peakVRAM 14.2GB +[05:30:52] step 380/4768 loss 4.4968 lr 4.56e-04 gnorm 0.53 58,616 tok/s peakVRAM 14.2GB +[05:33:51] step 400/4768 loss 4.3926 lr 4.80e-04 gnorm 0.41 58,609 tok/s peakVRAM 14.2GB +[05:36:50] step 420/4768 loss 4.3094 lr 5.04e-04 gnorm 0.46 58,611 tok/s peakVRAM 14.2GB +[05:39:49] step 440/4768 loss 4.2352 lr 5.28e-04 gnorm 0.44 58,612 tok/s peakVRAM 14.2GB +[05:42:48] step 460/4768 loss 4.1637 lr 5.52e-04 gnorm 0.39 58,612 tok/s peakVRAM 14.2GB +[05:45:47] step 480/4768 loss 4.1191 lr 5.76e-04 gnorm 0.34 58,613 tok/s peakVRAM 14.2GB +[05:48:46] step 500/4768 loss 4.0618 lr 6.00e-04 gnorm 0.32 58,609 tok/s peakVRAM 14.2GB +[05:48:47] ckpt @ 500 +[05:51:46] step 520/4768 loss 4.0278 lr 6.00e-04 gnorm 0.33 58,065 tok/s peakVRAM 14.2GB +[05:54:45] step 540/4768 loss 3.9765 lr 6.00e-04 gnorm 0.25 58,620 tok/s peakVRAM 14.2GB +[05:57:44] step 560/4768 loss 3.9295 lr 6.00e-04 gnorm 0.26 58,621 tok/s peakVRAM 14.2GB +[06:00:43] step 580/4768 loss 3.8875 lr 6.00e-04 gnorm 0.26 58,617 tok/s peakVRAM 14.2GB +[06:03:42] step 600/4768 loss 3.8484 lr 6.00e-04 gnorm 0.21 58,617 tok/s peakVRAM 14.2GB +[06:06:41] step 620/4768 loss 3.8276 lr 6.00e-04 gnorm 0.23 58,610 tok/s peakVRAM 14.2GB +[06:09:40] step 640/4768 loss 3.8017 lr 6.00e-04 gnorm 0.18 58,612 tok/s peakVRAM 14.2GB +[06:12:39] step 660/4768 loss 3.7725 lr 6.00e-04 gnorm 0.19 58,615 tok/s peakVRAM 14.2GB +[06:15:37] step 680/4768 loss 3.7515 lr 6.00e-04 gnorm 0.19 58,607 tok/s peakVRAM 14.2GB +[06:18:36] step 700/4768 loss 3.7217 lr 6.00e-04 gnorm 0.18 58,603 tok/s peakVRAM 14.2GB +[06:21:35] step 720/4768 loss 3.6930 lr 6.00e-04 gnorm 0.18 58,599 tok/s peakVRAM 14.2GB +[06:24:34] step 740/4768 loss 3.6890 lr 6.00e-04 gnorm 0.19 58,600 tok/s peakVRAM 14.2GB +[06:27:33] step 760/4768 loss 3.6650 lr 6.00e-04 gnorm 0.16 58,596 tok/s peakVRAM 14.2GB +[06:30:32] step 780/4768 loss 3.6402 lr 6.00e-04 gnorm 0.17 58,597 tok/s peakVRAM 14.2GB +[06:33:31] step 800/4768 loss 3.6266 lr 6.00e-04 gnorm 0.16 58,601 tok/s peakVRAM 14.2GB +[06:36:30] step 820/4768 loss 3.6111 lr 6.00e-04 gnorm 0.16 58,599 tok/s peakVRAM 14.2GB +[06:39:29] step 840/4768 loss 3.5983 lr 6.00e-04 gnorm 0.16 58,600 tok/s peakVRAM 14.2GB +[06:42:28] step 860/4768 loss 3.5771 lr 6.00e-04 gnorm 0.16 58,598 tok/s peakVRAM 14.2GB +[06:45:27] step 880/4768 loss 3.5720 lr 6.00e-04 gnorm 0.15 58,592 tok/s peakVRAM 14.2GB +[06:48:26] step 900/4768 loss 3.5545 lr 6.00e-04 gnorm 0.15 58,586 tok/s peakVRAM 14.2GB +[06:51:25] step 920/4768 loss 3.5473 lr 6.00e-04 gnorm 0.14 58,593 tok/s peakVRAM 14.2GB +[06:54:24] step 940/4768 loss 3.5419 lr 6.00e-04 gnorm 0.16 58,587 tok/s peakVRAM 14.2GB +[06:57:23] step 960/4768 loss 3.5175 lr 6.00e-04 gnorm 0.14 58,589 tok/s peakVRAM 14.2GB +[07:00:22] step 980/4768 loss 3.5143 lr 6.00e-04 gnorm 0.14 58,595 tok/s peakVRAM 14.2GB +[07:03:21] step 1000/4768 loss 3.5043 lr 6.00e-04 gnorm 0.15 58,588 tok/s peakVRAM 14.2GB +[07:03:23] ckpt @ 1000 +[07:06:22] step 1020/4768 loss 3.4993 lr 6.00e-04 gnorm 0.14 57,812 tok/s peakVRAM 14.2GB +[07:09:21] step 1040/4768 loss 3.4728 lr 6.00e-04 gnorm 0.13 58,591 tok/s peakVRAM 14.2GB +[07:12:20] step 1060/4768 loss 3.4845 lr 6.00e-04 gnorm 0.14 58,588 tok/s peakVRAM 14.2GB +[07:15:19] step 1080/4768 loss 3.4739 lr 6.00e-04 gnorm 0.16 58,587 tok/s peakVRAM 14.2GB +[07:18:18] step 1100/4768 loss 3.4584 lr 6.00e-04 gnorm 0.15 58,592 tok/s peakVRAM 14.2GB +[07:21:17] step 1120/4768 loss 3.4405 lr 6.00e-04 gnorm 0.12 58,595 tok/s peakVRAM 14.2GB +[07:24:16] step 1140/4768 loss 3.4338 lr 6.00e-04 gnorm 0.14 58,588 tok/s peakVRAM 14.2GB +[07:27:15] step 1160/4768 loss 3.4284 lr 6.00e-04 gnorm 0.12 58,585 tok/s peakVRAM 14.2GB +[07:30:14] step 1180/4768 loss 3.4317 lr 6.00e-04 gnorm 0.14 58,590 tok/s peakVRAM 14.2GB +[07:33:13] step 1200/4768 loss 3.4130 lr 6.00e-04 gnorm 0.14 58,581 tok/s peakVRAM 14.2GB +[07:36:12] step 1220/4768 loss 3.4098 lr 6.00e-04 gnorm 0.12 58,584 tok/s peakVRAM 14.2GB +[07:39:11] step 1240/4768 loss 3.4026 lr 6.00e-04 gnorm 0.12 58,586 tok/s peakVRAM 14.2GB +[07:42:10] step 1260/4768 loss 3.3888 lr 6.00e-04 gnorm 0.12 58,586 tok/s peakVRAM 14.2GB +[07:45:09] step 1280/4768 loss 3.3914 lr 6.00e-04 gnorm 0.12 58,589 tok/s peakVRAM 14.2GB +[07:48:08] step 1300/4768 loss 3.3844 lr 6.00e-04 gnorm 0.12 58,586 tok/s peakVRAM 14.2GB +[07:51:07] step 1320/4768 loss 3.3767 lr 6.00e-04 gnorm 0.14 58,590 tok/s peakVRAM 14.2GB +[07:54:06] step 1340/4768 loss 3.3768 lr 6.00e-04 gnorm 0.12 58,587 tok/s peakVRAM 14.2GB +[07:57:05] step 1360/4768 loss 3.3681 lr 6.00e-04 gnorm 0.13 58,590 tok/s peakVRAM 14.2GB +[08:00:04] step 1380/4768 loss 3.3594 lr 6.00e-04 gnorm 0.12 58,580 tok/s peakVRAM 14.2GB +[08:03:03] step 1400/4768 loss 3.3493 lr 6.00e-04 gnorm 0.12 58,579 tok/s peakVRAM 14.2GB +[08:06:02] step 1420/4768 loss 3.3370 lr 6.00e-04 gnorm 0.13 58,576 tok/s peakVRAM 14.2GB +[08:09:01] step 1440/4768 loss 3.3438 lr 6.00e-04 gnorm 0.13 58,580 tok/s peakVRAM 14.2GB +[08:12:00] step 1460/4768 loss 3.3417 lr 6.00e-04 gnorm 0.12 58,573 tok/s peakVRAM 14.2GB +[08:14:59] step 1480/4768 loss 3.3458 lr 6.00e-04 gnorm 0.11 58,573 tok/s peakVRAM 14.2GB +[08:17:58] step 1500/4768 loss 3.3185 lr 6.00e-04 gnorm 0.13 58,573 tok/s peakVRAM 14.2GB +[08:18:00] ckpt @ 1500 +[08:20:59] step 1520/4768 loss 3.3255 lr 6.00e-04 gnorm 0.11 57,778 tok/s peakVRAM 14.2GB +[08:23:58] step 1540/4768 loss 3.3099 lr 6.00e-04 gnorm 0.12 58,532 tok/s peakVRAM 14.2GB +[08:26:57] step 1560/4768 loss 3.3181 lr 6.00e-04 gnorm 0.12 58,532 tok/s peakVRAM 14.2GB +[08:29:57] step 1580/4768 loss 3.3065 lr 6.00e-04 gnorm 0.13 58,567 tok/s peakVRAM 14.2GB +[08:32:56] step 1600/4768 loss 3.2964 lr 6.00e-04 gnorm 0.12 58,572 tok/s peakVRAM 14.2GB +[08:35:55] step 1620/4768 loss 3.2961 lr 6.00e-04 gnorm 0.13 58,576 tok/s peakVRAM 14.2GB +[08:38:54] step 1640/4768 loss 3.2932 lr 6.00e-04 gnorm 0.11 58,575 tok/s peakVRAM 14.2GB +[08:41:53] step 1660/4768 loss 3.2939 lr 6.00e-04 gnorm 0.12 58,565 tok/s peakVRAM 14.2GB +[08:44:52] step 1680/4768 loss 3.2829 lr 6.00e-04 gnorm 0.11 58,562 tok/s peakVRAM 14.2GB +[08:47:51] step 1700/4768 loss 3.2761 lr 6.00e-04 gnorm 0.13 58,570 tok/s peakVRAM 14.2GB +[08:50:50] step 1720/4768 loss 3.2781 lr 6.00e-04 gnorm 0.12 58,566 tok/s peakVRAM 14.2GB +[08:53:49] step 1740/4768 loss 3.2715 lr 6.00e-04 gnorm 0.11 58,556 tok/s peakVRAM 14.2GB +[08:56:48] step 1760/4768 loss 3.2704 lr 6.00e-04 gnorm 0.13 58,551 tok/s peakVRAM 14.2GB +[08:59:47] step 1780/4768 loss 3.2580 lr 6.00e-04 gnorm 0.11 58,559 tok/s peakVRAM 14.2GB +[09:02:46] step 1800/4768 loss 3.2615 lr 6.00e-04 gnorm 0.12 58,560 tok/s peakVRAM 14.2GB +[09:05:45] step 1820/4768 loss 3.2574 lr 6.00e-04 gnorm 0.12 58,560 tok/s peakVRAM 14.2GB +[09:08:44] step 1840/4768 loss 3.2504 lr 6.00e-04 gnorm 0.11 58,555 tok/s peakVRAM 14.2GB +[09:11:43] step 1860/4768 loss 3.2495 lr 6.00e-04 gnorm 0.12 58,571 tok/s peakVRAM 14.2GB +[09:14:42] step 1880/4768 loss 3.2457 lr 6.00e-04 gnorm 0.11 58,554 tok/s peakVRAM 14.2GB +[09:17:41] step 1900/4768 loss 3.2398 lr 6.00e-04 gnorm 0.12 58,562 tok/s peakVRAM 14.2GB +[09:20:40] step 1920/4768 loss 3.2400 lr 6.00e-04 gnorm 0.11 58,556 tok/s peakVRAM 14.2GB +[09:23:39] step 1940/4768 loss 3.2326 lr 6.00e-04 gnorm 0.11 58,557 tok/s peakVRAM 14.2GB +[09:26:39] step 1960/4768 loss 3.2364 lr 6.00e-04 gnorm 0.12 58,559 tok/s peakVRAM 14.2GB +[09:29:38] step 1980/4768 loss 3.2304 lr 6.00e-04 gnorm 0.11 58,575 tok/s peakVRAM 14.2GB +[09:32:37] step 2000/4768 loss 3.2241 lr 6.00e-04 gnorm 0.11 58,571 tok/s peakVRAM 14.2GB +[09:32:39] ckpt @ 2000 +[09:35:38] step 2020/4768 loss 3.2240 lr 6.00e-04 gnorm 0.11 57,804 tok/s peakVRAM 14.2GB +[09:38:37] step 2040/4768 loss 3.2166 lr 6.00e-04 gnorm 0.11 58,583 tok/s peakVRAM 14.2GB +[09:41:36] step 2060/4768 loss 3.2176 lr 6.00e-04 gnorm 0.11 58,579 tok/s peakVRAM 14.2GB +[09:44:35] step 2080/4768 loss 3.2063 lr 6.00e-04 gnorm 0.11 58,579 tok/s peakVRAM 14.2GB +[09:47:34] step 2100/4768 loss 3.2144 lr 6.00e-04 gnorm 0.10 58,579 tok/s peakVRAM 14.2GB +[09:50:33] step 2120/4768 loss 3.2037 lr 6.00e-04 gnorm 0.11 58,579 tok/s peakVRAM 14.2GB +[09:53:32] step 2140/4768 loss 3.2170 lr 6.00e-04 gnorm 0.11 58,588 tok/s peakVRAM 14.2GB +[09:56:31] step 2160/4768 loss 3.1952 lr 6.00e-04 gnorm 0.11 58,585 tok/s peakVRAM 14.2GB +[09:59:30] step 2180/4768 loss 3.1976 lr 6.00e-04 gnorm 0.10 58,581 tok/s peakVRAM 14.2GB +[10:02:29] step 2200/4768 loss 3.1985 lr 6.00e-04 gnorm 0.10 58,583 tok/s peakVRAM 14.2GB +[10:05:28] step 2220/4768 loss 3.1905 lr 6.00e-04 gnorm 0.11 58,579 tok/s peakVRAM 14.2GB +[10:08:27] step 2240/4768 loss 3.1876 lr 6.00e-04 gnorm 0.10 58,566 tok/s peakVRAM 14.2GB +[10:11:26] step 2260/4768 loss 3.1840 lr 6.00e-04 gnorm 0.11 58,565 tok/s peakVRAM 14.2GB +[10:14:25] step 2280/4768 loss 3.1804 lr 6.00e-04 gnorm 0.10 58,565 tok/s peakVRAM 14.2GB +[10:17:24] step 2300/4768 loss 3.1842 lr 6.00e-04 gnorm 0.11 58,561 tok/s peakVRAM 14.2GB +[10:20:23] step 2320/4768 loss 3.1801 lr 6.00e-04 gnorm 0.12 58,573 tok/s peakVRAM 14.2GB +[10:23:22] step 2340/4768 loss 3.1748 lr 6.00e-04 gnorm 0.11 58,584 tok/s peakVRAM 14.2GB +[10:26:21] step 2360/4768 loss 3.1654 lr 6.00e-04 gnorm 0.11 58,582 tok/s peakVRAM 14.2GB +[10:29:20] step 2380/4768 loss 3.1644 lr 6.00e-04 gnorm 0.10 58,584 tok/s peakVRAM 14.2GB +[10:32:19] step 2400/4768 loss 3.1648 lr 6.00e-04 gnorm 0.12 58,586 tok/s peakVRAM 14.2GB +[10:35:18] step 2420/4768 loss 3.1676 lr 6.00e-04 gnorm 0.11 58,586 tok/s peakVRAM 14.2GB +[10:38:17] step 2440/4768 loss 3.1628 lr 6.00e-04 gnorm 0.10 58,580 tok/s peakVRAM 14.2GB +[10:41:16] step 2460/4768 loss 3.1619 lr 6.00e-04 gnorm 0.10 58,582 tok/s peakVRAM 14.2GB +[10:44:15] step 2480/4768 loss 3.1585 lr 6.00e-04 gnorm 0.11 58,589 tok/s peakVRAM 14.2GB +[10:47:14] step 2500/4768 loss 3.1526 lr 6.00e-04 gnorm 0.09 58,585 tok/s peakVRAM 14.2GB +[10:47:16] ckpt @ 2500 +[10:50:15] step 2520/4768 loss 3.1547 lr 6.00e-04 gnorm 0.12 57,830 tok/s peakVRAM 14.2GB +[10:53:14] step 2540/4768 loss 3.1546 lr 6.00e-04 gnorm 0.11 58,617 tok/s peakVRAM 14.2GB +[10:56:13] step 2560/4768 loss 3.1520 lr 6.00e-04 gnorm 0.11 58,598 tok/s peakVRAM 14.2GB +[10:59:12] step 2580/4768 loss 3.1469 lr 6.00e-04 gnorm 0.10 58,573 tok/s peakVRAM 14.2GB +[11:02:11] step 2600/4768 loss 3.1377 lr 6.00e-04 gnorm 0.11 58,616 tok/s peakVRAM 14.2GB +[11:05:10] step 2620/4768 loss 3.1459 lr 6.00e-04 gnorm 0.11 58,586 tok/s peakVRAM 14.2GB +[11:08:09] step 2640/4768 loss 3.1474 lr 6.00e-04 gnorm 0.11 58,584 tok/s peakVRAM 14.2GB +[11:11:08] step 2660/4768 loss 3.1453 lr 6.00e-04 gnorm 0.11 58,589 tok/s peakVRAM 14.2GB +[11:14:07] step 2680/4768 loss 3.1407 lr 6.00e-04 gnorm 0.10 58,578 tok/s peakVRAM 14.2GB +[11:17:06] step 2700/4768 loss 3.1430 lr 6.00e-04 gnorm 0.11 58,575 tok/s peakVRAM 14.2GB +[11:20:05] step 2720/4768 loss 3.1259 lr 6.00e-04 gnorm 0.10 58,573 tok/s peakVRAM 14.2GB +[11:23:04] step 2740/4768 loss 3.1274 lr 6.00e-04 gnorm 0.11 58,564 tok/s peakVRAM 14.2GB +[11:26:03] step 2760/4768 loss 3.1306 lr 6.00e-04 gnorm 0.11 58,568 tok/s peakVRAM 14.2GB +[11:29:02] step 2780/4768 loss 3.1218 lr 6.00e-04 gnorm 0.11 58,567 tok/s peakVRAM 14.2GB +[11:32:01] step 2800/4768 loss 3.1167 lr 6.00e-04 gnorm 0.11 58,560 tok/s peakVRAM 14.2GB +[11:35:00] step 2820/4768 loss 3.1136 lr 6.00e-04 gnorm 0.10 58,564 tok/s peakVRAM 14.2GB +[11:37:59] step 2840/4768 loss 3.1220 lr 6.00e-04 gnorm 0.10 58,585 tok/s peakVRAM 14.2GB +[11:40:58] step 2860/4768 loss 3.1131 lr 6.00e-04 gnorm 0.10 58,602 tok/s peakVRAM 14.2GB +[11:43:57] step 2880/4768 loss 3.1154 lr 6.00e-04 gnorm 0.10 58,558 tok/s peakVRAM 14.2GB +[11:46:56] step 2900/4768 loss 3.1166 lr 6.00e-04 gnorm 0.10 58,605 tok/s peakVRAM 14.2GB +[11:49:55] step 2920/4768 loss 3.1239 lr 6.00e-04 gnorm 0.10 58,617 tok/s peakVRAM 14.2GB +[11:52:54] step 2940/4768 loss 3.1144 lr 6.00e-04 gnorm 0.09 58,618 tok/s peakVRAM 14.2GB +[11:55:53] step 2960/4768 loss 3.1160 lr 6.00e-04 gnorm 0.11 58,609 tok/s peakVRAM 14.2GB +[11:58:52] step 2980/4768 loss 3.1031 lr 6.00e-04 gnorm 0.10 58,602 tok/s peakVRAM 14.2GB +[12:01:51] step 3000/4768 loss 3.1038 lr 6.00e-04 gnorm 0.09 58,570 tok/s peakVRAM 14.2GB +[12:01:53] ckpt @ 3000 +[12:04:52] step 3020/4768 loss 3.0994 lr 6.00e-04 gnorm 0.09 57,789 tok/s peakVRAM 14.2GB +[12:07:51] step 3040/4768 loss 3.0966 lr 6.00e-04 gnorm 0.10 58,562 tok/s peakVRAM 14.2GB +[12:10:50] step 3060/4768 loss 3.1002 lr 6.00e-04 gnorm 0.11 58,560 tok/s peakVRAM 14.2GB diff --git a/K12/run/train.out b/K12/run/train.out new file mode 100644 index 0000000000000000000000000000000000000000..fcf3b585978edecee6f731add0a45426c0355754 --- /dev/null +++ b/K12/run/train.out @@ -0,0 +1,167 @@ +[04:33:11] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[04:33:11] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[04:33:13] model GPT-ref: 184.6M param (L12 d1024 h16) +[04:33:14] optimizer=muon muon_lr=0.02 (mult=33.3x) +[04:33:14] r11 pamiec: act_ckpt=True loss_chunk=1024 +[04:33:14] r13: mixer_layout=AAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[04:33:15] torch.compile enabled +[04:33:15] START gpt-ref: steps=4768 (od 0) lr=0.0006->6e-05 +[04:37:14] step 20/4768 loss 10.4046 lr 2.40e-05 gnorm 2.87 43,863 tok/s peakVRAM 14.2GB +[04:40:12] step 40/4768 loss 9.5343 lr 4.80e-05 gnorm 1.38 58,709 tok/s peakVRAM 14.2GB +[04:43:11] step 60/4768 loss 8.5928 lr 7.20e-05 gnorm 0.59 58,729 tok/s peakVRAM 14.2GB +[04:46:09] step 80/4768 loss 7.9727 lr 9.60e-05 gnorm 0.86 58,711 tok/s peakVRAM 14.2GB +[04:49:08] step 100/4768 loss 7.5379 lr 1.20e-04 gnorm 0.89 58,683 tok/s peakVRAM 14.2GB +[04:52:07] step 120/4768 loss 7.0513 lr 1.44e-04 gnorm 0.87 58,663 tok/s peakVRAM 14.2GB +[04:55:06] step 140/4768 loss 6.6284 lr 1.68e-04 gnorm 0.71 58,646 tok/s peakVRAM 14.2GB +[04:58:05] step 160/4768 loss 6.2768 lr 1.92e-04 gnorm 0.77 58,628 tok/s peakVRAM 14.2GB +[05:01:03] step 180/4768 loss 6.0184 lr 2.16e-04 gnorm 0.77 58,626 tok/s peakVRAM 14.2GB +[05:04:02] step 200/4768 loss 5.8071 lr 2.40e-04 gnorm 0.83 58,612 tok/s peakVRAM 14.2GB +[05:07:01] step 220/4768 loss 5.6286 lr 2.64e-04 gnorm 0.80 58,612 tok/s peakVRAM 14.2GB +[05:10:00] step 240/4768 loss 5.4628 lr 2.88e-04 gnorm 0.86 58,623 tok/s peakVRAM 14.2GB +[05:12:59] step 260/4768 loss 5.3007 lr 3.12e-04 gnorm 0.86 58,626 tok/s peakVRAM 14.2GB +[05:15:58] step 280/4768 loss 5.1497 lr 3.36e-04 gnorm 0.74 58,625 tok/s peakVRAM 14.2GB +[05:18:57] step 300/4768 loss 5.0103 lr 3.60e-04 gnorm 0.83 58,621 tok/s peakVRAM 14.2GB +[05:21:56] step 320/4768 loss 4.8812 lr 3.84e-04 gnorm 0.59 58,614 tok/s peakVRAM 14.2GB +[05:24:54] step 340/4768 loss 4.7429 lr 4.08e-04 gnorm 0.76 58,616 tok/s peakVRAM 14.2GB +[05:27:53] step 360/4768 loss 4.6169 lr 4.32e-04 gnorm 0.55 58,614 tok/s peakVRAM 14.2GB +[05:30:52] step 380/4768 loss 4.4968 lr 4.56e-04 gnorm 0.53 58,616 tok/s peakVRAM 14.2GB +[05:33:51] step 400/4768 loss 4.3926 lr 4.80e-04 gnorm 0.41 58,609 tok/s peakVRAM 14.2GB +[05:36:50] step 420/4768 loss 4.3094 lr 5.04e-04 gnorm 0.46 58,611 tok/s peakVRAM 14.2GB +[05:39:49] step 440/4768 loss 4.2352 lr 5.28e-04 gnorm 0.44 58,612 tok/s peakVRAM 14.2GB +[05:42:48] step 460/4768 loss 4.1637 lr 5.52e-04 gnorm 0.39 58,612 tok/s peakVRAM 14.2GB +[05:45:47] step 480/4768 loss 4.1191 lr 5.76e-04 gnorm 0.34 58,613 tok/s peakVRAM 14.2GB +[05:48:46] step 500/4768 loss 4.0618 lr 6.00e-04 gnorm 0.32 58,609 tok/s peakVRAM 14.2GB +[05:48:47] ckpt @ 500 +[05:51:46] step 520/4768 loss 4.0278 lr 6.00e-04 gnorm 0.33 58,065 tok/s peakVRAM 14.2GB +[05:54:45] step 540/4768 loss 3.9765 lr 6.00e-04 gnorm 0.25 58,620 tok/s peakVRAM 14.2GB +[05:57:44] step 560/4768 loss 3.9295 lr 6.00e-04 gnorm 0.26 58,621 tok/s peakVRAM 14.2GB +[06:00:43] step 580/4768 loss 3.8875 lr 6.00e-04 gnorm 0.26 58,617 tok/s peakVRAM 14.2GB +[06:03:42] step 600/4768 loss 3.8484 lr 6.00e-04 gnorm 0.21 58,617 tok/s peakVRAM 14.2GB +[06:06:41] step 620/4768 loss 3.8276 lr 6.00e-04 gnorm 0.23 58,610 tok/s peakVRAM 14.2GB +[06:09:40] step 640/4768 loss 3.8017 lr 6.00e-04 gnorm 0.18 58,612 tok/s peakVRAM 14.2GB +[06:12:39] step 660/4768 loss 3.7725 lr 6.00e-04 gnorm 0.19 58,615 tok/s peakVRAM 14.2GB +[06:15:37] step 680/4768 loss 3.7515 lr 6.00e-04 gnorm 0.19 58,607 tok/s peakVRAM 14.2GB +[06:18:36] step 700/4768 loss 3.7217 lr 6.00e-04 gnorm 0.18 58,603 tok/s peakVRAM 14.2GB +[06:21:35] step 720/4768 loss 3.6930 lr 6.00e-04 gnorm 0.18 58,599 tok/s peakVRAM 14.2GB +[06:24:34] step 740/4768 loss 3.6890 lr 6.00e-04 gnorm 0.19 58,600 tok/s peakVRAM 14.2GB +[06:27:33] step 760/4768 loss 3.6650 lr 6.00e-04 gnorm 0.16 58,596 tok/s peakVRAM 14.2GB +[06:30:32] step 780/4768 loss 3.6402 lr 6.00e-04 gnorm 0.17 58,597 tok/s peakVRAM 14.2GB +[06:33:31] step 800/4768 loss 3.6266 lr 6.00e-04 gnorm 0.16 58,601 tok/s peakVRAM 14.2GB +[06:36:30] step 820/4768 loss 3.6111 lr 6.00e-04 gnorm 0.16 58,599 tok/s peakVRAM 14.2GB +[06:39:29] step 840/4768 loss 3.5983 lr 6.00e-04 gnorm 0.16 58,600 tok/s peakVRAM 14.2GB +[06:42:28] step 860/4768 loss 3.5771 lr 6.00e-04 gnorm 0.16 58,598 tok/s peakVRAM 14.2GB +[06:45:27] step 880/4768 loss 3.5720 lr 6.00e-04 gnorm 0.15 58,592 tok/s peakVRAM 14.2GB +[06:48:26] step 900/4768 loss 3.5545 lr 6.00e-04 gnorm 0.15 58,586 tok/s peakVRAM 14.2GB +[06:51:25] step 920/4768 loss 3.5473 lr 6.00e-04 gnorm 0.14 58,593 tok/s peakVRAM 14.2GB +[06:54:24] step 940/4768 loss 3.5419 lr 6.00e-04 gnorm 0.16 58,587 tok/s peakVRAM 14.2GB +[06:57:23] step 960/4768 loss 3.5175 lr 6.00e-04 gnorm 0.14 58,589 tok/s peakVRAM 14.2GB +[07:00:22] step 980/4768 loss 3.5143 lr 6.00e-04 gnorm 0.14 58,595 tok/s peakVRAM 14.2GB +[07:03:21] step 1000/4768 loss 3.5043 lr 6.00e-04 gnorm 0.15 58,588 tok/s peakVRAM 14.2GB +[07:03:23] ckpt @ 1000 +[07:06:22] step 1020/4768 loss 3.4993 lr 6.00e-04 gnorm 0.14 57,812 tok/s peakVRAM 14.2GB +[07:09:21] step 1040/4768 loss 3.4728 lr 6.00e-04 gnorm 0.13 58,591 tok/s peakVRAM 14.2GB +[07:12:20] step 1060/4768 loss 3.4845 lr 6.00e-04 gnorm 0.14 58,588 tok/s peakVRAM 14.2GB +[07:15:19] step 1080/4768 loss 3.4739 lr 6.00e-04 gnorm 0.16 58,587 tok/s peakVRAM 14.2GB +[07:18:18] step 1100/4768 loss 3.4584 lr 6.00e-04 gnorm 0.15 58,592 tok/s peakVRAM 14.2GB +[07:21:17] step 1120/4768 loss 3.4405 lr 6.00e-04 gnorm 0.12 58,595 tok/s peakVRAM 14.2GB +[07:24:16] step 1140/4768 loss 3.4338 lr 6.00e-04 gnorm 0.14 58,588 tok/s peakVRAM 14.2GB +[07:27:15] step 1160/4768 loss 3.4284 lr 6.00e-04 gnorm 0.12 58,585 tok/s peakVRAM 14.2GB +[07:30:14] step 1180/4768 loss 3.4317 lr 6.00e-04 gnorm 0.14 58,590 tok/s peakVRAM 14.2GB +[07:33:13] step 1200/4768 loss 3.4130 lr 6.00e-04 gnorm 0.14 58,581 tok/s peakVRAM 14.2GB +[07:36:12] step 1220/4768 loss 3.4098 lr 6.00e-04 gnorm 0.12 58,584 tok/s peakVRAM 14.2GB +[07:39:11] step 1240/4768 loss 3.4026 lr 6.00e-04 gnorm 0.12 58,586 tok/s peakVRAM 14.2GB +[07:42:10] step 1260/4768 loss 3.3888 lr 6.00e-04 gnorm 0.12 58,586 tok/s peakVRAM 14.2GB +[07:45:09] step 1280/4768 loss 3.3914 lr 6.00e-04 gnorm 0.12 58,589 tok/s peakVRAM 14.2GB +[07:48:08] step 1300/4768 loss 3.3844 lr 6.00e-04 gnorm 0.12 58,586 tok/s peakVRAM 14.2GB +[07:51:07] step 1320/4768 loss 3.3767 lr 6.00e-04 gnorm 0.14 58,590 tok/s peakVRAM 14.2GB +[07:54:06] step 1340/4768 loss 3.3768 lr 6.00e-04 gnorm 0.12 58,587 tok/s peakVRAM 14.2GB +[07:57:05] step 1360/4768 loss 3.3681 lr 6.00e-04 gnorm 0.13 58,590 tok/s peakVRAM 14.2GB +[08:00:04] step 1380/4768 loss 3.3594 lr 6.00e-04 gnorm 0.12 58,580 tok/s peakVRAM 14.2GB +[08:03:03] step 1400/4768 loss 3.3493 lr 6.00e-04 gnorm 0.12 58,579 tok/s peakVRAM 14.2GB +[08:06:02] step 1420/4768 loss 3.3370 lr 6.00e-04 gnorm 0.13 58,576 tok/s peakVRAM 14.2GB +[08:09:01] step 1440/4768 loss 3.3438 lr 6.00e-04 gnorm 0.13 58,580 tok/s peakVRAM 14.2GB +[08:12:00] step 1460/4768 loss 3.3417 lr 6.00e-04 gnorm 0.12 58,573 tok/s peakVRAM 14.2GB +[08:14:59] step 1480/4768 loss 3.3458 lr 6.00e-04 gnorm 0.11 58,573 tok/s peakVRAM 14.2GB +[08:17:58] step 1500/4768 loss 3.3185 lr 6.00e-04 gnorm 0.13 58,573 tok/s peakVRAM 14.2GB +[08:18:00] ckpt @ 1500 +[08:20:59] step 1520/4768 loss 3.3255 lr 6.00e-04 gnorm 0.11 57,778 tok/s peakVRAM 14.2GB +[08:23:58] step 1540/4768 loss 3.3099 lr 6.00e-04 gnorm 0.12 58,532 tok/s peakVRAM 14.2GB +[08:26:57] step 1560/4768 loss 3.3181 lr 6.00e-04 gnorm 0.12 58,532 tok/s peakVRAM 14.2GB +[08:29:57] step 1580/4768 loss 3.3065 lr 6.00e-04 gnorm 0.13 58,567 tok/s peakVRAM 14.2GB +[08:32:56] step 1600/4768 loss 3.2964 lr 6.00e-04 gnorm 0.12 58,572 tok/s peakVRAM 14.2GB +[08:35:55] step 1620/4768 loss 3.2961 lr 6.00e-04 gnorm 0.13 58,576 tok/s peakVRAM 14.2GB +[08:38:54] step 1640/4768 loss 3.2932 lr 6.00e-04 gnorm 0.11 58,575 tok/s peakVRAM 14.2GB +[08:41:53] step 1660/4768 loss 3.2939 lr 6.00e-04 gnorm 0.12 58,565 tok/s peakVRAM 14.2GB +[08:44:52] step 1680/4768 loss 3.2829 lr 6.00e-04 gnorm 0.11 58,562 tok/s peakVRAM 14.2GB +[08:47:51] step 1700/4768 loss 3.2761 lr 6.00e-04 gnorm 0.13 58,570 tok/s peakVRAM 14.2GB +[08:50:50] step 1720/4768 loss 3.2781 lr 6.00e-04 gnorm 0.12 58,566 tok/s peakVRAM 14.2GB +[08:53:49] step 1740/4768 loss 3.2715 lr 6.00e-04 gnorm 0.11 58,556 tok/s peakVRAM 14.2GB +[08:56:48] step 1760/4768 loss 3.2704 lr 6.00e-04 gnorm 0.13 58,551 tok/s peakVRAM 14.2GB +[08:59:47] step 1780/4768 loss 3.2580 lr 6.00e-04 gnorm 0.11 58,559 tok/s peakVRAM 14.2GB +[09:02:46] step 1800/4768 loss 3.2615 lr 6.00e-04 gnorm 0.12 58,560 tok/s peakVRAM 14.2GB +[09:05:45] step 1820/4768 loss 3.2574 lr 6.00e-04 gnorm 0.12 58,560 tok/s peakVRAM 14.2GB +[09:08:44] step 1840/4768 loss 3.2504 lr 6.00e-04 gnorm 0.11 58,555 tok/s peakVRAM 14.2GB +[09:11:43] step 1860/4768 loss 3.2495 lr 6.00e-04 gnorm 0.12 58,571 tok/s peakVRAM 14.2GB +[09:14:42] step 1880/4768 loss 3.2457 lr 6.00e-04 gnorm 0.11 58,554 tok/s peakVRAM 14.2GB +[09:17:41] step 1900/4768 loss 3.2398 lr 6.00e-04 gnorm 0.12 58,562 tok/s peakVRAM 14.2GB +[09:20:40] step 1920/4768 loss 3.2400 lr 6.00e-04 gnorm 0.11 58,556 tok/s peakVRAM 14.2GB +[09:23:39] step 1940/4768 loss 3.2326 lr 6.00e-04 gnorm 0.11 58,557 tok/s peakVRAM 14.2GB +[09:26:39] step 1960/4768 loss 3.2364 lr 6.00e-04 gnorm 0.12 58,559 tok/s peakVRAM 14.2GB +[09:29:38] step 1980/4768 loss 3.2304 lr 6.00e-04 gnorm 0.11 58,575 tok/s peakVRAM 14.2GB +[09:32:37] step 2000/4768 loss 3.2241 lr 6.00e-04 gnorm 0.11 58,571 tok/s peakVRAM 14.2GB +[09:32:39] ckpt @ 2000 +[09:35:38] step 2020/4768 loss 3.2240 lr 6.00e-04 gnorm 0.11 57,804 tok/s peakVRAM 14.2GB +[09:38:37] step 2040/4768 loss 3.2166 lr 6.00e-04 gnorm 0.11 58,583 tok/s peakVRAM 14.2GB +[09:41:36] step 2060/4768 loss 3.2176 lr 6.00e-04 gnorm 0.11 58,579 tok/s peakVRAM 14.2GB +[09:44:35] step 2080/4768 loss 3.2063 lr 6.00e-04 gnorm 0.11 58,579 tok/s peakVRAM 14.2GB +[09:47:34] step 2100/4768 loss 3.2144 lr 6.00e-04 gnorm 0.10 58,579 tok/s peakVRAM 14.2GB +[09:50:33] step 2120/4768 loss 3.2037 lr 6.00e-04 gnorm 0.11 58,579 tok/s peakVRAM 14.2GB +[09:53:32] step 2140/4768 loss 3.2170 lr 6.00e-04 gnorm 0.11 58,588 tok/s peakVRAM 14.2GB +[09:56:31] step 2160/4768 loss 3.1952 lr 6.00e-04 gnorm 0.11 58,585 tok/s peakVRAM 14.2GB +[09:59:30] step 2180/4768 loss 3.1976 lr 6.00e-04 gnorm 0.10 58,581 tok/s peakVRAM 14.2GB +[10:02:29] step 2200/4768 loss 3.1985 lr 6.00e-04 gnorm 0.10 58,583 tok/s peakVRAM 14.2GB +[10:05:28] step 2220/4768 loss 3.1905 lr 6.00e-04 gnorm 0.11 58,579 tok/s peakVRAM 14.2GB +[10:08:27] step 2240/4768 loss 3.1876 lr 6.00e-04 gnorm 0.10 58,566 tok/s peakVRAM 14.2GB +[10:11:26] step 2260/4768 loss 3.1840 lr 6.00e-04 gnorm 0.11 58,565 tok/s peakVRAM 14.2GB +[10:14:25] step 2280/4768 loss 3.1804 lr 6.00e-04 gnorm 0.10 58,565 tok/s peakVRAM 14.2GB +[10:17:24] step 2300/4768 loss 3.1842 lr 6.00e-04 gnorm 0.11 58,561 tok/s peakVRAM 14.2GB +[10:20:23] step 2320/4768 loss 3.1801 lr 6.00e-04 gnorm 0.12 58,573 tok/s peakVRAM 14.2GB +[10:23:22] step 2340/4768 loss 3.1748 lr 6.00e-04 gnorm 0.11 58,584 tok/s peakVRAM 14.2GB +[10:26:21] step 2360/4768 loss 3.1654 lr 6.00e-04 gnorm 0.11 58,582 tok/s peakVRAM 14.2GB +[10:29:20] step 2380/4768 loss 3.1644 lr 6.00e-04 gnorm 0.10 58,584 tok/s peakVRAM 14.2GB +[10:32:19] step 2400/4768 loss 3.1648 lr 6.00e-04 gnorm 0.12 58,586 tok/s peakVRAM 14.2GB +[10:35:18] step 2420/4768 loss 3.1676 lr 6.00e-04 gnorm 0.11 58,586 tok/s peakVRAM 14.2GB +[10:38:17] step 2440/4768 loss 3.1628 lr 6.00e-04 gnorm 0.10 58,580 tok/s peakVRAM 14.2GB +[10:41:16] step 2460/4768 loss 3.1619 lr 6.00e-04 gnorm 0.10 58,582 tok/s peakVRAM 14.2GB +[10:44:15] step 2480/4768 loss 3.1585 lr 6.00e-04 gnorm 0.11 58,589 tok/s peakVRAM 14.2GB +[10:47:14] step 2500/4768 loss 3.1526 lr 6.00e-04 gnorm 0.09 58,585 tok/s peakVRAM 14.2GB +[10:47:16] ckpt @ 2500 +[10:50:15] step 2520/4768 loss 3.1547 lr 6.00e-04 gnorm 0.12 57,830 tok/s peakVRAM 14.2GB +[10:53:14] step 2540/4768 loss 3.1546 lr 6.00e-04 gnorm 0.11 58,617 tok/s peakVRAM 14.2GB +[10:56:13] step 2560/4768 loss 3.1520 lr 6.00e-04 gnorm 0.11 58,598 tok/s peakVRAM 14.2GB +[10:59:12] step 2580/4768 loss 3.1469 lr 6.00e-04 gnorm 0.10 58,573 tok/s peakVRAM 14.2GB +[11:02:11] step 2600/4768 loss 3.1377 lr 6.00e-04 gnorm 0.11 58,616 tok/s peakVRAM 14.2GB +[11:05:10] step 2620/4768 loss 3.1459 lr 6.00e-04 gnorm 0.11 58,586 tok/s peakVRAM 14.2GB +[11:08:09] step 2640/4768 loss 3.1474 lr 6.00e-04 gnorm 0.11 58,584 tok/s peakVRAM 14.2GB +[11:11:08] step 2660/4768 loss 3.1453 lr 6.00e-04 gnorm 0.11 58,589 tok/s peakVRAM 14.2GB +[11:14:07] step 2680/4768 loss 3.1407 lr 6.00e-04 gnorm 0.10 58,578 tok/s peakVRAM 14.2GB +[11:17:06] step 2700/4768 loss 3.1430 lr 6.00e-04 gnorm 0.11 58,575 tok/s peakVRAM 14.2GB +[11:20:05] step 2720/4768 loss 3.1259 lr 6.00e-04 gnorm 0.10 58,573 tok/s peakVRAM 14.2GB +[11:23:04] step 2740/4768 loss 3.1274 lr 6.00e-04 gnorm 0.11 58,564 tok/s peakVRAM 14.2GB +[11:26:03] step 2760/4768 loss 3.1306 lr 6.00e-04 gnorm 0.11 58,568 tok/s peakVRAM 14.2GB +[11:29:02] step 2780/4768 loss 3.1218 lr 6.00e-04 gnorm 0.11 58,567 tok/s peakVRAM 14.2GB +[11:32:01] step 2800/4768 loss 3.1167 lr 6.00e-04 gnorm 0.11 58,560 tok/s peakVRAM 14.2GB +[11:35:00] step 2820/4768 loss 3.1136 lr 6.00e-04 gnorm 0.10 58,564 tok/s peakVRAM 14.2GB +[11:37:59] step 2840/4768 loss 3.1220 lr 6.00e-04 gnorm 0.10 58,585 tok/s peakVRAM 14.2GB +[11:40:58] step 2860/4768 loss 3.1131 lr 6.00e-04 gnorm 0.10 58,602 tok/s peakVRAM 14.2GB +[11:43:57] step 2880/4768 loss 3.1154 lr 6.00e-04 gnorm 0.10 58,558 tok/s peakVRAM 14.2GB +[11:46:56] step 2900/4768 loss 3.1166 lr 6.00e-04 gnorm 0.10 58,605 tok/s peakVRAM 14.2GB +[11:49:55] step 2920/4768 loss 3.1239 lr 6.00e-04 gnorm 0.10 58,617 tok/s peakVRAM 14.2GB +[11:52:54] step 2940/4768 loss 3.1144 lr 6.00e-04 gnorm 0.09 58,618 tok/s peakVRAM 14.2GB +[11:55:53] step 2960/4768 loss 3.1160 lr 6.00e-04 gnorm 0.11 58,609 tok/s peakVRAM 14.2GB +[11:58:52] step 2980/4768 loss 3.1031 lr 6.00e-04 gnorm 0.10 58,602 tok/s peakVRAM 14.2GB +[12:01:51] step 3000/4768 loss 3.1038 lr 6.00e-04 gnorm 0.09 58,570 tok/s peakVRAM 14.2GB +[12:01:53] ckpt @ 3000 +[12:04:52] step 3020/4768 loss 3.0994 lr 6.00e-04 gnorm 0.09 57,789 tok/s peakVRAM 14.2GB +[12:07:51] step 3040/4768 loss 3.0966 lr 6.00e-04 gnorm 0.10 58,562 tok/s peakVRAM 14.2GB +[12:10:50] step 3060/4768 loss 3.1002 lr 6.00e-04 gnorm 0.11 58,560 tok/s peakVRAM 14.2GB diff --git a/K22/run/args.txt b/K22/run/args.txt new file mode 100644 index 0000000000000000000000000000000000000000..7bb72322b184b61d3cc3fa35fbe56d4b6361b081 --- /dev/null +++ b/K22/run/args.txt @@ -0,0 +1 @@ +--data-dir /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2 --out-dir /home/ubuntu/kolektyw/dowody400m/wyniki/K22 --run-id dowody_K22 --vocab 32768 --dtype uint16 --block 4096 --n-layer 22 --n-embd 768 --n-head 12 --value-residual --batch 128 --grad-accum 8 --norm rmsnorm --pos rope --rope-theta 500000 --ffn swiglu --ffn-mult 2.667 --qk-norm --attn-bias 0 --optimizer muon --muon-lr 0.02 --lr 6e-4 --min-lr 6e-05 --wd 0.1 --schedule wsd --decay-frac 0.2 --warmup 500 --act-ckpt --loss-chunk 1024 --sampler random --seed 1 --data-seed 20261022 --compile --log-every 20 --eval-every 4769 --eval-iters 1 --steps 4768 --ckpt-every 4768 \ No newline at end of file diff --git a/K22/run/metrics.jsonl b/K22/run/metrics.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..cd83cffaea635b12cb1afb1e8d8e0bb8f404c75d --- /dev/null +++ b/K22/run/metrics.jsonl @@ -0,0 +1,238 @@ +{"step": 20, "loss": 10.37869439125061, "lr": 2.3999999999999997e-05, "tok_s": 36393.553282902314} +{"step": 40, "loss": 9.627096700668336, "lr": 4.7999999999999994e-05, "tok_s": 50497.672795545426} +{"step": 60, "loss": 8.837148857116699, "lr": 7.199999999999999e-05, "tok_s": 50510.12726419128} +{"step": 80, "loss": 8.161912059783935, "lr": 9.599999999999999e-05, "tok_s": 50497.238002564685} +{"step": 100, "loss": 7.683762717247009, "lr": 0.00011999999999999999, "tok_s": 50465.52822745929} +{"step": 120, "loss": 7.22345232963562, "lr": 0.00014399999999999998, "tok_s": 50440.79953964529} +{"step": 140, "loss": 6.816223430633545, "lr": 0.000168, "tok_s": 50430.647907341336} +{"step": 160, "loss": 6.451836967468262, "lr": 0.00019199999999999998, "tok_s": 50417.94875199477} +{"step": 180, "loss": 6.164482140541077, "lr": 0.00021599999999999996, "tok_s": 50412.281978101135} +{"step": 200, "loss": 5.928548049926758, "lr": 0.00023999999999999998, "tok_s": 50410.515272681805} +{"step": 220, "loss": 5.729818272590637, "lr": 0.00026399999999999997, "tok_s": 50409.25054367114} +{"step": 240, "loss": 5.550185561180115, "lr": 0.00028799999999999995, "tok_s": 50408.0463106913} +{"step": 260, "loss": 5.381895279884338, "lr": 0.000312, "tok_s": 50407.06276837466} +{"step": 280, "loss": 5.224881482124329, "lr": 0.000336, "tok_s": 50410.07464126373} +{"step": 300, "loss": 5.07756621837616, "lr": 0.00035999999999999997, "tok_s": 50409.40983744519} +{"step": 320, "loss": 4.930981469154358, "lr": 0.00038399999999999996, "tok_s": 50409.892404924765} +{"step": 340, "loss": 4.773558759689331, "lr": 0.000408, "tok_s": 50408.03908881801} +{"step": 360, "loss": 4.648498702049255, "lr": 0.00043199999999999993, "tok_s": 50400.943471663995} +{"step": 380, "loss": 4.530560612678528, "lr": 0.00045599999999999997, "tok_s": 50403.91250220873} +{"step": 400, "loss": 4.440827441215515, "lr": 0.00047999999999999996, "tok_s": 50401.6025649675} +{"step": 420, "loss": 4.364287209510803, "lr": 0.000504, "tok_s": 50404.13403394797} +{"step": 440, "loss": 4.294577264785767, "lr": 0.0005279999999999999, "tok_s": 50403.16883988858} +{"step": 460, "loss": 4.225187301635742, "lr": 0.000552, "tok_s": 50392.21966179442} +{"step": 480, "loss": 4.177277779579162, "lr": 0.0005759999999999999, "tok_s": 50388.11379174276} +{"step": 500, "loss": 4.125906372070313, "lr": 0.0006, "tok_s": 50391.59355088377} +{"step": 520, "loss": 4.089947938919067, "lr": 0.0006, "tok_s": 50395.15474966302} +{"step": 540, "loss": 4.037944316864014, "lr": 0.0006, "tok_s": 50396.28745206674} +{"step": 560, "loss": 3.9871296882629395, "lr": 0.0006, "tok_s": 50388.8140580344} +{"step": 580, "loss": 3.942476987838745, "lr": 0.0006, "tok_s": 50386.95542198973} +{"step": 600, "loss": 3.899268078804016, "lr": 0.0006, "tok_s": 50382.43288655501} +{"step": 620, "loss": 3.873553729057312, "lr": 0.0006, "tok_s": 50383.500545223214} +{"step": 640, "loss": 3.846368408203125, "lr": 0.0006, "tok_s": 50384.20264578695} +{"step": 660, "loss": 3.813744640350342, "lr": 0.0006, "tok_s": 50378.349674127} +{"step": 680, "loss": 3.788252055644989, "lr": 0.0006, "tok_s": 50378.65350300499} +{"step": 700, "loss": 3.7559040665626524, "lr": 0.0006, "tok_s": 50379.44509110364} +{"step": 720, "loss": 3.723824405670166, "lr": 0.0006, "tok_s": 50376.78597997029} +{"step": 740, "loss": 3.718950796127319, "lr": 0.0006, "tok_s": 50373.67873035097} +{"step": 760, "loss": 3.6942316770553587, "lr": 0.0006, "tok_s": 50378.68010623984} +{"step": 780, "loss": 3.667694294452667, "lr": 0.0006, "tok_s": 50381.48179766727} +{"step": 800, "loss": 3.6529730796813964, "lr": 0.0006, "tok_s": 50377.88306012204} +{"step": 820, "loss": 3.636118769645691, "lr": 0.0006, "tok_s": 50379.51341918606} +{"step": 840, "loss": 3.621321976184845, "lr": 0.0006, "tok_s": 50379.233067473695} +{"step": 860, "loss": 3.599413800239563, "lr": 0.0006, "tok_s": 50382.71471680499} +{"step": 880, "loss": 3.5940122485160826, "lr": 0.0006, "tok_s": 50370.448236529955} +{"step": 900, "loss": 3.5738147139549254, "lr": 0.0006, "tok_s": 50371.711019226} +{"step": 920, "loss": 3.566121315956116, "lr": 0.0006, "tok_s": 50372.40546419553} +{"step": 940, "loss": 3.5609045147895815, "lr": 0.0006, "tok_s": 50369.0759717683} +{"step": 960, "loss": 3.534505832195282, "lr": 0.0006, "tok_s": 50374.16540284457} +{"step": 980, "loss": 3.5308042645454405, "lr": 0.0006, "tok_s": 50370.44696737549} +{"step": 1000, "loss": 3.5204467296600344, "lr": 0.0006, "tok_s": 50368.656600237875} +{"step": 1020, "loss": 3.5146281719207764, "lr": 0.0006, "tok_s": 50360.423738338155} +{"step": 1040, "loss": 3.486905074119568, "lr": 0.0006, "tok_s": 50363.738476024184} +{"step": 1060, "loss": 3.498551595211029, "lr": 0.0006, "tok_s": 50367.67886292377} +{"step": 1080, "loss": 3.488401412963867, "lr": 0.0006, "tok_s": 50371.610001203284} +{"step": 1100, "loss": 3.471914494037628, "lr": 0.0006, "tok_s": 50366.92980914957} +{"step": 1120, "loss": 3.4547880053520204, "lr": 0.0006, "tok_s": 50363.24122025229} +{"step": 1140, "loss": 3.4481158375740053, "lr": 0.0006, "tok_s": 50367.25397723054} +{"step": 1160, "loss": 3.4420236945152283, "lr": 0.0006, "tok_s": 50367.32319515499} +{"step": 1180, "loss": 3.443655323982239, "lr": 0.0006, "tok_s": 50365.537029822786} +{"step": 1200, "loss": 3.4253238439559937, "lr": 0.0006, "tok_s": 50364.3213885592} +{"step": 1220, "loss": 3.422685420513153, "lr": 0.0006, "tok_s": 50357.29002336478} +{"step": 1240, "loss": 3.413462817668915, "lr": 0.0006, "tok_s": 50364.86180730632} +{"step": 1260, "loss": 3.3992380857467652, "lr": 0.0006, "tok_s": 50365.81434521177} +{"step": 1280, "loss": 3.4017250537872314, "lr": 0.0006, "tok_s": 50360.62176375309} +{"step": 1300, "loss": 3.395185101032257, "lr": 0.0006, "tok_s": 50359.43484555891} +{"step": 1320, "loss": 3.386663091182709, "lr": 0.0006, "tok_s": 50361.95371831432} +{"step": 1340, "loss": 3.386570703983307, "lr": 0.0006, "tok_s": 50360.018292791196} +{"step": 1360, "loss": 3.37717227935791, "lr": 0.0006, "tok_s": 50360.13858216995} +{"step": 1380, "loss": 3.3683578252792357, "lr": 0.0006, "tok_s": 50362.59056938791} +{"step": 1400, "loss": 3.3574086308479307, "lr": 0.0006, "tok_s": 50367.077530065406} +{"step": 1420, "loss": 3.3459409832954408, "lr": 0.0006, "tok_s": 50368.306052530184} +{"step": 1440, "loss": 3.351805567741394, "lr": 0.0006, "tok_s": 50366.085895769036} +{"step": 1460, "loss": 3.349338412284851, "lr": 0.0006, "tok_s": 50362.918776298226} +{"step": 1480, "loss": 3.3528262257575987, "lr": 0.0006, "tok_s": 50366.965513578914} +{"step": 1500, "loss": 3.326026129722595, "lr": 0.0006, "tok_s": 50361.33625977772} +{"step": 1520, "loss": 3.3332749724388124, "lr": 0.0006, "tok_s": 50362.476150693976} +{"step": 1540, "loss": 3.3186570405960083, "lr": 0.0006, "tok_s": 50362.37470834519} +{"step": 1560, "loss": 3.326381599903107, "lr": 0.0006, "tok_s": 50363.40616366726} +{"step": 1580, "loss": 3.3143098831176756, "lr": 0.0006, "tok_s": 50366.3740032681} +{"step": 1600, "loss": 3.303647184371948, "lr": 0.0006, "tok_s": 50366.45988820335} +{"step": 1620, "loss": 3.3041441798210145, "lr": 0.0006, "tok_s": 50364.35253297463} +{"step": 1640, "loss": 3.3000163197517396, "lr": 0.0006, "tok_s": 50371.227682520854} +{"step": 1660, "loss": 3.3013440489768984, "lr": 0.0006, "tok_s": 50375.75189930125} +{"step": 1680, "loss": 3.2892932176589964, "lr": 0.0006, "tok_s": 50375.30852879603} +{"step": 1700, "loss": 3.281483030319214, "lr": 0.0006, "tok_s": 50372.8753230105} +{"step": 1720, "loss": 3.2839611768722534, "lr": 0.0006, "tok_s": 50365.161263114795} +{"step": 1740, "loss": 3.2778034806251526, "lr": 0.0006, "tok_s": 50365.621008536684} +{"step": 1760, "loss": 3.2767387986183167, "lr": 0.0006, "tok_s": 50362.62620999453} +{"step": 1780, "loss": 3.2648613452911377, "lr": 0.0006, "tok_s": 50359.492912887785} +{"step": 1800, "loss": 3.268283021450043, "lr": 0.0006, "tok_s": 50353.9341067899} +{"step": 1820, "loss": 3.263661098480225, "lr": 0.0006, "tok_s": 50363.4779086227} +{"step": 1840, "loss": 3.2567388653755187, "lr": 0.0006, "tok_s": 50360.198957736284} +{"step": 1860, "loss": 3.2560648560523986, "lr": 0.0006, "tok_s": 50360.82371206727} +{"step": 1880, "loss": 3.250918185710907, "lr": 0.0006, "tok_s": 50367.90942092212} +{"step": 1900, "loss": 3.2456185102462767, "lr": 0.0006, "tok_s": 50366.967186324015} +{"step": 1920, "loss": 3.24539510011673, "lr": 0.0006, "tok_s": 50364.89202957227} +{"step": 1940, "loss": 3.2371082663536073, "lr": 0.0006, "tok_s": 50360.922264816654} +{"step": 1960, "loss": 3.2425138473510744, "lr": 0.0006, "tok_s": 50367.19612272798} +{"step": 1980, "loss": 3.2364680767059326, "lr": 0.0006, "tok_s": 50368.85544031163} +{"step": 2000, "loss": 3.229717528820038, "lr": 0.0006, "tok_s": 50370.214597807935} +{"step": 2020, "loss": 3.229957103729248, "lr": 0.0006, "tok_s": 50368.395578155156} +{"step": 2040, "loss": 3.222858762741089, "lr": 0.0006, "tok_s": 50365.71588867058} +{"step": 2060, "loss": 3.2233617067337037, "lr": 0.0006, "tok_s": 50366.5096081587} +{"step": 2080, "loss": 3.21190664768219, "lr": 0.0006, "tok_s": 50369.32344462723} +{"step": 2100, "loss": 3.2206087470054627, "lr": 0.0006, "tok_s": 50368.81315710099} +{"step": 2120, "loss": 3.209831988811493, "lr": 0.0006, "tok_s": 50366.00099264635} +{"step": 2140, "loss": 3.222468078136444, "lr": 0.0006, "tok_s": 50370.005997758206} +{"step": 2160, "loss": 3.20089076757431, "lr": 0.0006, "tok_s": 50368.9796945229} +{"step": 2180, "loss": 3.202888286113739, "lr": 0.0006, "tok_s": 50368.795736208114} +{"step": 2200, "loss": 3.2039575815200805, "lr": 0.0006, "tok_s": 50374.92148034183} +{"step": 2220, "loss": 3.196550261974335, "lr": 0.0006, "tok_s": 50369.89650649877} +{"step": 2240, "loss": 3.192943048477173, "lr": 0.0006, "tok_s": 50365.647771042815} +{"step": 2260, "loss": 3.1894694447517393, "lr": 0.0006, "tok_s": 50364.36118420798} +{"step": 2280, "loss": 3.185406744480133, "lr": 0.0006, "tok_s": 50366.768534199626} +{"step": 2300, "loss": 3.188775873184204, "lr": 0.0006, "tok_s": 50365.339023428074} +{"step": 2320, "loss": 3.185637879371643, "lr": 0.0006, "tok_s": 50365.96684690598} +{"step": 2340, "loss": 3.180560517311096, "lr": 0.0006, "tok_s": 50371.4503116318} +{"step": 2360, "loss": 3.1701487064361573, "lr": 0.0006, "tok_s": 50373.50783466527} +{"step": 2380, "loss": 3.1692240476608275, "lr": 0.0006, "tok_s": 50366.82823349739} +{"step": 2400, "loss": 3.169828176498413, "lr": 0.0006, "tok_s": 50362.98994314746} +{"step": 2420, "loss": 3.1725643992424013, "lr": 0.0006, "tok_s": 50363.46568197598} +{"step": 2440, "loss": 3.1676795840263368, "lr": 0.0006, "tok_s": 50364.831008316876} +{"step": 2460, "loss": 3.1653196454048156, "lr": 0.0006, "tok_s": 50360.98812064299} +{"step": 2480, "loss": 3.1631489992141724, "lr": 0.0006, "tok_s": 50367.85814076448} +{"step": 2500, "loss": 3.1574719667434694, "lr": 0.0006, "tok_s": 50370.88592556927} +{"step": 2520, "loss": 3.1593925595283507, "lr": 0.0006, "tok_s": 50366.13423069093} +{"step": 2540, "loss": 3.1598495960235597, "lr": 0.0006, "tok_s": 50369.71940625962} +{"step": 2560, "loss": 3.1563698768615724, "lr": 0.0006, "tok_s": 50377.445826522744} +{"step": 2580, "loss": 3.1529322624206544, "lr": 0.0006, "tok_s": 50373.92815243526} +{"step": 2600, "loss": 3.1432958245277405, "lr": 0.0006, "tok_s": 50370.01482399743} +{"step": 2620, "loss": 3.151778995990753, "lr": 0.0006, "tok_s": 50367.93993526121} +{"step": 2640, "loss": 3.15283340215683, "lr": 0.0006, "tok_s": 50372.823051937885} +{"step": 2660, "loss": 3.15077862739563, "lr": 0.0006, "tok_s": 50367.684861904236} +{"step": 2680, "loss": 3.1459423899650574, "lr": 0.0006, "tok_s": 50369.369593681135} +{"step": 2700, "loss": 3.1483611345291136, "lr": 0.0006, "tok_s": 50367.77998054509} +{"step": 2720, "loss": 3.1315477013587953, "lr": 0.0006, "tok_s": 50371.11322457831} +{"step": 2740, "loss": 3.1327906966209413, "lr": 0.0006, "tok_s": 50371.19560655305} +{"step": 2760, "loss": 3.1356467247009276, "lr": 0.0006, "tok_s": 50367.30923619158} +{"step": 2780, "loss": 3.1263054609298706, "lr": 0.0006, "tok_s": 50365.03783521227} +{"step": 2800, "loss": 3.1220290780067446, "lr": 0.0006, "tok_s": 50367.36686022686} +{"step": 2820, "loss": 3.1191172122955324, "lr": 0.0006, "tok_s": 50368.037073781525} +{"step": 2840, "loss": 3.1271515130996703, "lr": 0.0006, "tok_s": 50368.817252743116} +{"step": 2860, "loss": 3.118371379375458, "lr": 0.0006, "tok_s": 50365.231801855974} +{"step": 2880, "loss": 3.120266079902649, "lr": 0.0006, "tok_s": 50370.91950108056} +{"step": 2900, "loss": 3.1216858506202696, "lr": 0.0006, "tok_s": 50368.59712734414} +{"step": 2920, "loss": 3.128999102115631, "lr": 0.0006, "tok_s": 50367.76665583817} +{"step": 2940, "loss": 3.1193459153175356, "lr": 0.0006, "tok_s": 50374.91790299865} +{"step": 2960, "loss": 3.121028017997742, "lr": 0.0006, "tok_s": 50373.98919573658} +{"step": 2980, "loss": 3.1082852602005007, "lr": 0.0006, "tok_s": 50368.03234374989} +{"step": 3000, "loss": 3.1095697045326234, "lr": 0.0006, "tok_s": 50368.92350893083} +{"step": 3020, "loss": 3.104125189781189, "lr": 0.0006, "tok_s": 50373.03248304779} +{"step": 3040, "loss": 3.1018962025642396, "lr": 0.0006, "tok_s": 50368.885840426454} +{"step": 3060, "loss": 3.1055035829544066, "lr": 0.0006, "tok_s": 50370.910386074785} +{"step": 3080, "loss": 3.097902750968933, "lr": 0.0006, "tok_s": 50369.13521504951} +{"step": 3100, "loss": 3.0915641665458677, "lr": 0.0006, "tok_s": 50369.54715295462} +{"step": 3120, "loss": 3.1072227120399476, "lr": 0.0006, "tok_s": 50364.592461614484} +{"step": 3140, "loss": 3.093355858325958, "lr": 0.0006, "tok_s": 50368.89414710558} +{"step": 3160, "loss": 3.089010214805603, "lr": 0.0006, "tok_s": 50365.959406370726} +{"step": 3180, "loss": 3.095322680473328, "lr": 0.0006, "tok_s": 50363.213652891805} +{"step": 3200, "loss": 3.0964788436889648, "lr": 0.0006, "tok_s": 50369.25797080211} +{"step": 3220, "loss": 3.10577677488327, "lr": 0.0006, "tok_s": 50366.56192380935} +{"step": 3240, "loss": 3.087931716442108, "lr": 0.0006, "tok_s": 50369.38920705465} +{"step": 3260, "loss": 3.0842342734336854, "lr": 0.0006, "tok_s": 50364.850906536085} +{"step": 3280, "loss": 3.0802849888801576, "lr": 0.0006, "tok_s": 50360.093891657736} +{"step": 3300, "loss": 3.0906956791877747, "lr": 0.0006, "tok_s": 50361.87015551386} +{"step": 3320, "loss": 3.0849864363670347, "lr": 0.0006, "tok_s": 50359.74052224592} +{"step": 3340, "loss": 3.085504877567291, "lr": 0.0006, "tok_s": 50356.23731273821} +{"step": 3360, "loss": 3.075977456569672, "lr": 0.0006, "tok_s": 50359.37706668138} +{"step": 3380, "loss": 3.0861980319023132, "lr": 0.0006, "tok_s": 50372.40604112878} +{"step": 3400, "loss": 3.0712910890579224, "lr": 0.0006, "tok_s": 50370.44217920239} +{"step": 3420, "loss": 3.0787760853767394, "lr": 0.0006, "tok_s": 50371.70450006096} +{"step": 3440, "loss": 3.0708940505981444, "lr": 0.0006, "tok_s": 50365.13398203566} +{"step": 3460, "loss": 3.0802491188049315, "lr": 0.0006, "tok_s": 50362.65383438291} +{"step": 3480, "loss": 3.079214537143707, "lr": 0.0006, "tok_s": 50363.616612271275} +{"step": 3500, "loss": 3.0654635429382324, "lr": 0.0006, "tok_s": 50372.11924924044} +{"step": 3520, "loss": 3.069408428668976, "lr": 0.0006, "tok_s": 50368.90401129059} +{"step": 3540, "loss": 3.071292555332184, "lr": 0.0006, "tok_s": 50368.42695834645} +{"step": 3560, "loss": 3.0752272963523866, "lr": 0.0006, "tok_s": 50361.02819936066} +{"step": 3580, "loss": 3.068881928920746, "lr": 0.0006, "tok_s": 50364.0289213076} +{"step": 3600, "loss": 3.071395516395569, "lr": 0.0006, "tok_s": 50363.11676366342} +{"step": 3620, "loss": 3.0628724098205566, "lr": 0.0006, "tok_s": 50362.50458230678} +{"step": 3640, "loss": 3.070946991443634, "lr": 0.0006, "tok_s": 50359.04550382977} +{"step": 3660, "loss": 3.0597490549087523, "lr": 0.0006, "tok_s": 50360.805893033226} +{"step": 3680, "loss": 3.066377651691437, "lr": 0.0006, "tok_s": 50360.72025804551} +{"step": 3700, "loss": 3.06362179517746, "lr": 0.0006, "tok_s": 50359.1441649455} +{"step": 3720, "loss": 3.056973707675934, "lr": 0.0006, "tok_s": 50362.49864223086} +{"step": 3740, "loss": 3.0527987122535705, "lr": 0.0006, "tok_s": 50365.69177927745} +{"step": 3760, "loss": 3.0512983441352843, "lr": 0.0006, "tok_s": 50365.42346290032} +{"step": 3780, "loss": 3.0522939562797546, "lr": 0.0006, "tok_s": 50360.74597734455} +{"step": 3800, "loss": 3.053767907619476, "lr": 0.0006, "tok_s": 50362.55325637969} +{"step": 3820, "loss": 3.04669303894043, "lr": 0.0005609064981513237, "tok_s": 50363.96876725364} +{"step": 3840, "loss": 3.0445285201072694, "lr": 0.0005125842723878462, "tok_s": 50368.51533071292} +{"step": 3860, "loss": 3.0375494599342345, "lr": 0.0004827194944539712, "tok_s": 50367.89003943295} +{"step": 3880, "loss": 3.0345866322517394, "lr": 0.0004590463745471515, "tok_s": 50368.46491458596} +{"step": 3900, "loss": 3.0270234823226927, "lr": 0.0004388133626026285, "tok_s": 50370.28722748961} +{"step": 3920, "loss": 3.0222036361694338, "lr": 0.000420851068597545, "tok_s": 50370.17571591987} +{"step": 3940, "loss": 3.013981819152832, "lr": 0.00040453249075661876, "tok_s": 50372.37880989385} +{"step": 3960, "loss": 3.01424001455307, "lr": 0.00038947504965686095, "tok_s": 50371.7039808355} +{"step": 3980, "loss": 3.0062872052192686, "lr": 0.000375424929557729, "tok_s": 50378.14360348276} +{"step": 4000, "loss": 3.0049508571624757, "lr": 0.00036220351177672794, "tok_s": 50379.4737150075} +{"step": 4020, "loss": 3.002310800552368, "lr": 0.0003496794507866152, "tok_s": 50376.86745724493} +{"step": 4040, "loss": 2.993136239051819, "lr": 0.0003377528171635386, "tok_s": 50372.42121447796} +{"step": 4060, "loss": 2.992757511138916, "lr": 0.00032634548705926626, "tok_s": 50371.321833799} +{"step": 4080, "loss": 2.9841498017311094, "lr": 0.00031539501058484577, "tok_s": 50366.712699622156} +{"step": 4100, "loss": 2.98936128616333, "lr": 0.00030485053354754334, "tok_s": 50373.20879885303} +{"step": 4120, "loss": 2.977723276615143, "lr": 0.00029466998985632013, "tok_s": 50378.21083160903} +{"step": 4140, "loss": 2.9701634645462036, "lr": 0.0002848181118123862, "tok_s": 50375.00179759999} +{"step": 4160, "loss": 2.988078737258911, "lr": 0.000275264984658754, "tok_s": 50375.43229554511} +{"step": 4180, "loss": 2.9809946775436402, "lr": 0.0002659849737873323, "tok_s": 50370.452159371445} +{"step": 4200, "loss": 2.9741480231285093, "lr": 0.0002569559134994179, "tok_s": 50372.19724984914} +{"step": 4220, "loss": 2.9675135731697084, "lr": 0.00024815848336191376, "tok_s": 50371.85409513139} +{"step": 4240, "loss": 2.974126946926117, "lr": 0.00023957572171486765, "tok_s": 50366.25916362766} +{"step": 4260, "loss": 2.9661878824234007, "lr": 0.0002311926411735683, "tok_s": 50362.92667730802} +{"step": 4280, "loss": 2.9653461813926696, "lr": 0.0002229959211471338, "tok_s": 50365.8565657094} +{"step": 4300, "loss": 2.9630411863327026, "lr": 0.00021497365931813446, "tok_s": 50366.109428750446} +{"step": 4320, "loss": 2.9579756259918213, "lr": 0.00020711516882674415, "tok_s": 50366.67936043967} +{"step": 4340, "loss": 2.957149875164032, "lr": 0.00019941081128766396, "tok_s": 50362.23606920707} +{"step": 4360, "loss": 2.951228082180023, "lr": 0.00019185185819315875, "tok_s": 50365.82778424227} +{"step": 4380, "loss": 2.958230221271515, "lr": 0.00018443037501815989, "tok_s": 50367.44265420377} +{"step": 4400, "loss": 2.9549214482307433, "lr": 0.00017713912364145464, "tok_s": 50367.89811505163} +{"step": 4420, "loss": 2.9486687541007996, "lr": 0.00016997147966456131, "tok_s": 50481.350438563284} +{"step": 4440, "loss": 2.9544939279556273, "lr": 0.00016292136193923112, "tok_s": 50542.27322178109} +{"step": 4460, "loss": 2.9502453923225405, "lr": 0.00015598317216998812, "tok_s": 50556.77445617177} +{"step": 4480, "loss": 2.9451239466667176, "lr": 0.00014915174288526275, "tok_s": 50558.55189331388} +{"step": 4500, "loss": 2.9426443696022035, "lr": 0.00014242229240208165, "tok_s": 50564.620810065586} +{"step": 4520, "loss": 2.944092261791229, "lr": 0.00013579038566856907, "tok_s": 50564.0560981644} +{"step": 4540, "loss": 2.936077344417572, "lr": 0.00012925190007297358, "tok_s": 50562.4841099072} +{"step": 4560, "loss": 2.938389253616333, "lr": 0.0001228029954703451, "tok_s": 50564.53134127988} +{"step": 4580, "loss": 2.9421222805976868, "lr": 0.00011644008780788571, "tok_s": 50561.70216324656} +{"step": 4600, "loss": 2.940841829776764, "lr": 0.00011015982583457378, "tok_s": 50556.39164617272} +{"step": 4620, "loss": 2.9273490071296693, "lr": 0.00010395907046536478, "tok_s": 50486.87667619206} +{"step": 4640, "loss": 2.9367039680480955, "lr": 9.783487643927846e-05, "tok_s": 50538.73162112472} +{"step": 4660, "loss": 2.9295316457748415, "lr": 9.178447596720887e-05, "tok_s": 50553.55117600604} +{"step": 4680, "loss": 2.927152705192566, "lr": 8.580526411184025e-05, "tok_s": 50522.58288650106} +{"step": 4700, "loss": 2.923268437385559, "lr": 7.98947856805662e-05, "tok_s": 50501.552229447654} +{"step": 4720, "loss": 2.9241204977035524, "lr": 7.405072344432876e-05, "tok_s": 50526.73577754924} +{"step": 4740, "loss": 2.9223405003547667, "lr": 6.827088752203553e-05, "tok_s": 50521.72945441874} +{"step": 4760, "loss": 2.9310539960861206, "lr": 6.255320579263513e-05, "tok_s": 50546.140050920156} diff --git a/K22/run/srodowisko.txt b/K22/run/srodowisko.txt new file mode 100644 index 0000000000000000000000000000000000000000..ffa82f39baf44aa9d6c87ec72168789e3a9072d6 --- /dev/null +++ b/K22/run/srodowisko.txt @@ -0,0 +1,63 @@ +data 2026-10-22T11:59:50Z ARM K22 GPU 3 MINI 0 DATA_SEED 20261022 +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +a607076bd6e11cad879b8213b37492a4b4f5cbf8fa96205b7ebaeace8a25213e /home/ubuntu/kolektyw/dowody400m/kod/ramie.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 diff --git a/K22/run/srodowisko_K22_mini0.txt b/K22/run/srodowisko_K22_mini0.txt new file mode 100644 index 0000000000000000000000000000000000000000..ffa82f39baf44aa9d6c87ec72168789e3a9072d6 --- /dev/null +++ b/K22/run/srodowisko_K22_mini0.txt @@ -0,0 +1,63 @@ +data 2026-10-22T11:59:50Z ARM K22 GPU 3 MINI 0 DATA_SEED 20261022 +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +a607076bd6e11cad879b8213b37492a4b4f5cbf8fa96205b7ebaeace8a25213e /home/ubuntu/kolektyw/dowody400m/kod/ramie.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 diff --git a/K22/run/train.log b/K22/run/train.log new file mode 100644 index 0000000000000000000000000000000000000000..ab79fa1ba1aba8a847ff15062d3c70195ca8321c --- /dev/null +++ b/K22/run/train.log @@ -0,0 +1,248 @@ +[13:59:56] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[13:59:56] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[13:59:58] model GPT-ref: 180.9M param (L22 d768 h12) +[13:59:59] optimizer=muon muon_lr=0.02 (mult=33.3x) +[13:59:59] r11 pamiec: act_ckpt=True loss_chunk=1024 +[13:59:59] r13: mixer_layout=AAAAAAAAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[14:00:00] torch.compile enabled +[14:00:00] START gpt-ref: steps=4768 (od 0) lr=0.0006->6e-05 +[14:04:48] step 20/4768 loss 10.3787 lr 2.40e-05 gnorm 2.38 36,394 tok/s peakVRAM 13.7GB +[14:08:15] step 40/4768 loss 9.6271 lr 4.80e-05 gnorm 1.40 50,498 tok/s peakVRAM 13.7GB +[14:11:43] step 60/4768 loss 8.8371 lr 7.20e-05 gnorm 1.11 50,510 tok/s peakVRAM 13.7GB +[14:15:11] step 80/4768 loss 8.1619 lr 9.60e-05 gnorm 1.04 50,497 tok/s peakVRAM 13.7GB +[14:18:38] step 100/4768 loss 7.6838 lr 1.20e-04 gnorm 0.69 50,466 tok/s peakVRAM 13.7GB +[14:22:06] step 120/4768 loss 7.2235 lr 1.44e-04 gnorm 0.83 50,441 tok/s peakVRAM 13.7GB +[14:25:34] step 140/4768 loss 6.8162 lr 1.68e-04 gnorm 0.88 50,431 tok/s peakVRAM 13.7GB +[14:29:02] step 160/4768 loss 6.4518 lr 1.92e-04 gnorm 0.96 50,418 tok/s peakVRAM 13.7GB +[14:32:30] step 180/4768 loss 6.1645 lr 2.16e-04 gnorm 0.75 50,412 tok/s peakVRAM 13.7GB +[14:35:58] step 200/4768 loss 5.9285 lr 2.40e-04 gnorm 0.78 50,411 tok/s peakVRAM 13.7GB +[14:39:26] step 220/4768 loss 5.7298 lr 2.64e-04 gnorm 0.55 50,409 tok/s peakVRAM 13.7GB +[14:42:54] step 240/4768 loss 5.5502 lr 2.88e-04 gnorm 0.49 50,408 tok/s peakVRAM 13.7GB +[14:46:22] step 260/4768 loss 5.3819 lr 3.12e-04 gnorm 0.59 50,407 tok/s peakVRAM 13.7GB +[14:49:50] step 280/4768 loss 5.2249 lr 3.36e-04 gnorm 0.58 50,410 tok/s peakVRAM 13.7GB +[14:53:18] step 300/4768 loss 5.0776 lr 3.60e-04 gnorm 0.52 50,409 tok/s peakVRAM 13.7GB +[14:56:46] step 320/4768 loss 4.9310 lr 3.84e-04 gnorm 0.63 50,410 tok/s peakVRAM 13.7GB +[15:00:14] step 340/4768 loss 4.7736 lr 4.08e-04 gnorm 0.59 50,408 tok/s peakVRAM 13.7GB +[15:03:42] step 360/4768 loss 4.6485 lr 4.32e-04 gnorm 0.58 50,401 tok/s peakVRAM 13.7GB +[15:07:10] step 380/4768 loss 4.5306 lr 4.56e-04 gnorm 0.55 50,404 tok/s peakVRAM 13.7GB +[15:10:38] step 400/4768 loss 4.4408 lr 4.80e-04 gnorm 0.48 50,402 tok/s peakVRAM 13.7GB +[15:14:06] step 420/4768 loss 4.3643 lr 5.04e-04 gnorm 0.43 50,404 tok/s peakVRAM 13.7GB +[15:17:34] step 440/4768 loss 4.2946 lr 5.28e-04 gnorm 0.50 50,403 tok/s peakVRAM 13.7GB +[15:21:02] step 460/4768 loss 4.2252 lr 5.52e-04 gnorm 0.44 50,392 tok/s peakVRAM 13.7GB +[15:24:31] step 480/4768 loss 4.1773 lr 5.76e-04 gnorm 0.43 50,388 tok/s peakVRAM 13.7GB +[15:27:59] step 500/4768 loss 4.1259 lr 6.00e-04 gnorm 0.40 50,392 tok/s peakVRAM 13.7GB +[15:31:27] step 520/4768 loss 4.0899 lr 6.00e-04 gnorm 0.33 50,395 tok/s peakVRAM 13.7GB +[15:34:55] step 540/4768 loss 4.0379 lr 6.00e-04 gnorm 0.25 50,396 tok/s peakVRAM 13.7GB +[15:38:23] step 560/4768 loss 3.9871 lr 6.00e-04 gnorm 0.33 50,389 tok/s peakVRAM 13.7GB +[15:41:51] step 580/4768 loss 3.9425 lr 6.00e-04 gnorm 0.29 50,387 tok/s peakVRAM 13.7GB +[15:45:19] step 600/4768 loss 3.8993 lr 6.00e-04 gnorm 0.26 50,382 tok/s peakVRAM 13.7GB +[15:48:47] step 620/4768 loss 3.8736 lr 6.00e-04 gnorm 0.25 50,384 tok/s peakVRAM 13.7GB +[15:52:15] step 640/4768 loss 3.8464 lr 6.00e-04 gnorm 0.28 50,384 tok/s peakVRAM 13.7GB +[15:55:44] step 660/4768 loss 3.8137 lr 6.00e-04 gnorm 0.29 50,378 tok/s peakVRAM 13.7GB +[15:59:12] step 680/4768 loss 3.7883 lr 6.00e-04 gnorm 0.22 50,379 tok/s peakVRAM 13.7GB +[16:02:40] step 700/4768 loss 3.7559 lr 6.00e-04 gnorm 0.25 50,379 tok/s peakVRAM 13.7GB +[16:06:08] step 720/4768 loss 3.7238 lr 6.00e-04 gnorm 0.25 50,377 tok/s peakVRAM 13.7GB +[16:09:36] step 740/4768 loss 3.7190 lr 6.00e-04 gnorm 0.28 50,374 tok/s peakVRAM 13.7GB +[16:13:04] step 760/4768 loss 3.6942 lr 6.00e-04 gnorm 0.22 50,379 tok/s peakVRAM 13.7GB +[16:16:32] step 780/4768 loss 3.6677 lr 6.00e-04 gnorm 0.22 50,381 tok/s peakVRAM 13.7GB +[16:20:01] step 800/4768 loss 3.6530 lr 6.00e-04 gnorm 0.20 50,378 tok/s peakVRAM 13.7GB +[16:23:29] step 820/4768 loss 3.6361 lr 6.00e-04 gnorm 0.21 50,380 tok/s peakVRAM 13.7GB +[16:26:57] step 840/4768 loss 3.6213 lr 6.00e-04 gnorm 0.21 50,379 tok/s peakVRAM 13.7GB +[16:30:25] step 860/4768 loss 3.5994 lr 6.00e-04 gnorm 0.18 50,383 tok/s peakVRAM 13.7GB +[16:33:53] step 880/4768 loss 3.5940 lr 6.00e-04 gnorm 0.21 50,370 tok/s peakVRAM 13.7GB +[16:37:21] step 900/4768 loss 3.5738 lr 6.00e-04 gnorm 0.18 50,372 tok/s peakVRAM 13.7GB +[16:40:49] step 920/4768 loss 3.5661 lr 6.00e-04 gnorm 0.19 50,372 tok/s peakVRAM 13.7GB +[16:44:18] step 940/4768 loss 3.5609 lr 6.00e-04 gnorm 0.18 50,369 tok/s peakVRAM 13.7GB +[16:47:46] step 960/4768 loss 3.5345 lr 6.00e-04 gnorm 0.17 50,374 tok/s peakVRAM 13.7GB +[16:51:14] step 980/4768 loss 3.5308 lr 6.00e-04 gnorm 0.17 50,370 tok/s peakVRAM 13.7GB +[16:54:42] step 1000/4768 loss 3.5204 lr 6.00e-04 gnorm 0.17 50,369 tok/s peakVRAM 13.7GB +[16:58:10] step 1020/4768 loss 3.5146 lr 6.00e-04 gnorm 0.19 50,360 tok/s peakVRAM 13.7GB +[17:01:39] step 1040/4768 loss 3.4869 lr 6.00e-04 gnorm 0.18 50,364 tok/s peakVRAM 13.7GB +[17:05:07] step 1060/4768 loss 3.4986 lr 6.00e-04 gnorm 0.18 50,368 tok/s peakVRAM 13.7GB +[17:08:35] step 1080/4768 loss 3.4884 lr 6.00e-04 gnorm 0.16 50,372 tok/s peakVRAM 13.7GB +[17:12:03] step 1100/4768 loss 3.4719 lr 6.00e-04 gnorm 0.21 50,367 tok/s peakVRAM 13.7GB +[17:15:31] step 1120/4768 loss 3.4548 lr 6.00e-04 gnorm 0.18 50,363 tok/s peakVRAM 13.7GB +[17:18:59] step 1140/4768 loss 3.4481 lr 6.00e-04 gnorm 0.16 50,367 tok/s peakVRAM 13.7GB +[17:22:28] step 1160/4768 loss 3.4420 lr 6.00e-04 gnorm 0.17 50,367 tok/s peakVRAM 13.7GB +[17:25:56] step 1180/4768 loss 3.4437 lr 6.00e-04 gnorm 0.16 50,366 tok/s peakVRAM 13.7GB +[17:29:24] step 1200/4768 loss 3.4253 lr 6.00e-04 gnorm 0.19 50,364 tok/s peakVRAM 13.7GB +[17:32:52] step 1220/4768 loss 3.4227 lr 6.00e-04 gnorm 0.14 50,357 tok/s peakVRAM 13.7GB +[17:36:20] step 1240/4768 loss 3.4135 lr 6.00e-04 gnorm 0.15 50,365 tok/s peakVRAM 13.7GB +[17:39:49] step 1260/4768 loss 3.3992 lr 6.00e-04 gnorm 0.15 50,366 tok/s peakVRAM 13.7GB +[17:43:17] step 1280/4768 loss 3.4017 lr 6.00e-04 gnorm 0.15 50,361 tok/s peakVRAM 13.7GB +[17:46:45] step 1300/4768 loss 3.3952 lr 6.00e-04 gnorm 0.16 50,359 tok/s peakVRAM 13.7GB +[17:50:13] step 1320/4768 loss 3.3867 lr 6.00e-04 gnorm 0.16 50,362 tok/s peakVRAM 13.7GB +[17:53:42] step 1340/4768 loss 3.3866 lr 6.00e-04 gnorm 0.15 50,360 tok/s peakVRAM 13.7GB +[17:57:10] step 1360/4768 loss 3.3772 lr 6.00e-04 gnorm 0.13 50,360 tok/s peakVRAM 13.7GB +[18:00:38] step 1380/4768 loss 3.3684 lr 6.00e-04 gnorm 0.13 50,363 tok/s peakVRAM 13.7GB +[18:04:06] step 1400/4768 loss 3.3574 lr 6.00e-04 gnorm 0.14 50,367 tok/s peakVRAM 13.7GB +[18:07:34] step 1420/4768 loss 3.3459 lr 6.00e-04 gnorm 0.16 50,368 tok/s peakVRAM 13.7GB +[18:11:02] step 1440/4768 loss 3.3518 lr 6.00e-04 gnorm 0.16 50,366 tok/s peakVRAM 13.7GB +[18:14:31] step 1460/4768 loss 3.3493 lr 6.00e-04 gnorm 0.13 50,363 tok/s peakVRAM 13.7GB +[18:17:59] step 1480/4768 loss 3.3528 lr 6.00e-04 gnorm 0.13 50,367 tok/s peakVRAM 13.7GB +[18:21:27] step 1500/4768 loss 3.3260 lr 6.00e-04 gnorm 0.16 50,361 tok/s peakVRAM 13.7GB +[18:24:55] step 1520/4768 loss 3.3333 lr 6.00e-04 gnorm 0.14 50,362 tok/s peakVRAM 13.7GB +[18:28:23] step 1540/4768 loss 3.3187 lr 6.00e-04 gnorm 0.15 50,362 tok/s peakVRAM 13.7GB +[18:31:52] step 1560/4768 loss 3.3264 lr 6.00e-04 gnorm 0.16 50,363 tok/s peakVRAM 13.7GB +[18:35:20] step 1580/4768 loss 3.3143 lr 6.00e-04 gnorm 0.12 50,366 tok/s peakVRAM 13.7GB +[18:38:48] step 1600/4768 loss 3.3036 lr 6.00e-04 gnorm 0.15 50,366 tok/s peakVRAM 13.7GB +[18:42:16] step 1620/4768 loss 3.3041 lr 6.00e-04 gnorm 0.14 50,364 tok/s peakVRAM 13.7GB +[18:45:44] step 1640/4768 loss 3.3000 lr 6.00e-04 gnorm 0.14 50,371 tok/s peakVRAM 13.7GB +[18:49:13] step 1660/4768 loss 3.3013 lr 6.00e-04 gnorm 0.13 50,376 tok/s peakVRAM 13.7GB +[18:52:41] step 1680/4768 loss 3.2893 lr 6.00e-04 gnorm 0.13 50,375 tok/s peakVRAM 13.7GB +[18:56:09] step 1700/4768 loss 3.2815 lr 6.00e-04 gnorm 0.12 50,373 tok/s peakVRAM 13.7GB +[18:59:37] step 1720/4768 loss 3.2840 lr 6.00e-04 gnorm 0.15 50,365 tok/s peakVRAM 13.7GB +[19:03:05] step 1740/4768 loss 3.2778 lr 6.00e-04 gnorm 0.13 50,366 tok/s peakVRAM 13.7GB +[19:06:34] step 1760/4768 loss 3.2767 lr 6.00e-04 gnorm 0.13 50,363 tok/s peakVRAM 13.7GB +[19:10:02] step 1780/4768 loss 3.2649 lr 6.00e-04 gnorm 0.14 50,359 tok/s peakVRAM 13.7GB +[19:13:30] step 1800/4768 loss 3.2683 lr 6.00e-04 gnorm 0.13 50,354 tok/s peakVRAM 13.7GB +[19:16:58] step 1820/4768 loss 3.2637 lr 6.00e-04 gnorm 0.12 50,363 tok/s peakVRAM 13.7GB +[19:20:26] step 1840/4768 loss 3.2567 lr 6.00e-04 gnorm 0.12 50,360 tok/s peakVRAM 13.7GB +[19:23:55] step 1860/4768 loss 3.2561 lr 6.00e-04 gnorm 0.13 50,361 tok/s peakVRAM 13.7GB +[19:27:23] step 1880/4768 loss 3.2509 lr 6.00e-04 gnorm 0.13 50,368 tok/s peakVRAM 13.7GB +[19:30:51] step 1900/4768 loss 3.2456 lr 6.00e-04 gnorm 0.12 50,367 tok/s peakVRAM 13.7GB +[19:34:19] step 1920/4768 loss 3.2454 lr 6.00e-04 gnorm 0.11 50,365 tok/s peakVRAM 13.7GB +[19:37:47] step 1940/4768 loss 3.2371 lr 6.00e-04 gnorm 0.13 50,361 tok/s peakVRAM 13.7GB +[19:41:16] step 1960/4768 loss 3.2425 lr 6.00e-04 gnorm 0.13 50,367 tok/s peakVRAM 13.7GB +[19:44:44] step 1980/4768 loss 3.2365 lr 6.00e-04 gnorm 0.12 50,369 tok/s peakVRAM 13.7GB +[19:48:12] step 2000/4768 loss 3.2297 lr 6.00e-04 gnorm 0.13 50,370 tok/s peakVRAM 13.7GB +[19:51:40] step 2020/4768 loss 3.2300 lr 6.00e-04 gnorm 0.13 50,368 tok/s peakVRAM 13.7GB +[19:55:08] step 2040/4768 loss 3.2229 lr 6.00e-04 gnorm 0.13 50,366 tok/s peakVRAM 13.7GB +[19:58:36] step 2060/4768 loss 3.2234 lr 6.00e-04 gnorm 0.12 50,367 tok/s peakVRAM 13.7GB +[20:02:05] step 2080/4768 loss 3.2119 lr 6.00e-04 gnorm 0.13 50,369 tok/s peakVRAM 13.7GB +[20:05:33] step 2100/4768 loss 3.2206 lr 6.00e-04 gnorm 0.11 50,369 tok/s peakVRAM 13.7GB +[20:09:01] step 2120/4768 loss 3.2098 lr 6.00e-04 gnorm 0.13 50,366 tok/s peakVRAM 13.7GB +[20:12:29] step 2140/4768 loss 3.2225 lr 6.00e-04 gnorm 0.13 50,370 tok/s peakVRAM 13.7GB +[20:15:57] step 2160/4768 loss 3.2009 lr 6.00e-04 gnorm 0.13 50,369 tok/s peakVRAM 13.7GB +[20:19:26] step 2180/4768 loss 3.2029 lr 6.00e-04 gnorm 0.11 50,369 tok/s peakVRAM 13.7GB +[20:22:54] step 2200/4768 loss 3.2040 lr 6.00e-04 gnorm 0.12 50,375 tok/s peakVRAM 13.7GB +[20:26:22] step 2220/4768 loss 3.1966 lr 6.00e-04 gnorm 0.15 50,370 tok/s peakVRAM 13.7GB +[20:29:50] step 2240/4768 loss 3.1929 lr 6.00e-04 gnorm 0.12 50,366 tok/s peakVRAM 13.7GB +[20:33:18] step 2260/4768 loss 3.1895 lr 6.00e-04 gnorm 0.12 50,364 tok/s peakVRAM 13.7GB +[20:36:46] step 2280/4768 loss 3.1854 lr 6.00e-04 gnorm 0.12 50,367 tok/s peakVRAM 13.7GB +[20:40:15] step 2300/4768 loss 3.1888 lr 6.00e-04 gnorm 0.12 50,365 tok/s peakVRAM 13.7GB +[20:43:43] step 2320/4768 loss 3.1856 lr 6.00e-04 gnorm 0.13 50,366 tok/s peakVRAM 13.7GB +[20:47:11] step 2340/4768 loss 3.1806 lr 6.00e-04 gnorm 0.14 50,371 tok/s peakVRAM 13.7GB +[20:50:39] step 2360/4768 loss 3.1701 lr 6.00e-04 gnorm 0.12 50,374 tok/s peakVRAM 13.7GB +[20:54:07] step 2380/4768 loss 3.1692 lr 6.00e-04 gnorm 0.13 50,367 tok/s peakVRAM 13.7GB +[20:57:36] step 2400/4768 loss 3.1698 lr 6.00e-04 gnorm 0.12 50,363 tok/s peakVRAM 13.7GB +[21:01:04] step 2420/4768 loss 3.1726 lr 6.00e-04 gnorm 0.12 50,363 tok/s peakVRAM 13.7GB +[21:04:32] step 2440/4768 loss 3.1677 lr 6.00e-04 gnorm 0.14 50,365 tok/s peakVRAM 13.7GB +[21:08:00] step 2460/4768 loss 3.1653 lr 6.00e-04 gnorm 0.15 50,361 tok/s peakVRAM 13.7GB +[21:11:28] step 2480/4768 loss 3.1631 lr 6.00e-04 gnorm 0.13 50,368 tok/s peakVRAM 13.7GB +[21:14:57] step 2500/4768 loss 3.1575 lr 6.00e-04 gnorm 0.11 50,371 tok/s peakVRAM 13.7GB +[21:18:25] step 2520/4768 loss 3.1594 lr 6.00e-04 gnorm 0.12 50,366 tok/s peakVRAM 13.7GB +[21:21:53] step 2540/4768 loss 3.1598 lr 6.00e-04 gnorm 0.13 50,370 tok/s peakVRAM 13.7GB +[21:25:21] step 2560/4768 loss 3.1564 lr 6.00e-04 gnorm 0.13 50,377 tok/s peakVRAM 13.7GB +[21:28:49] step 2580/4768 loss 3.1529 lr 6.00e-04 gnorm 0.14 50,374 tok/s peakVRAM 13.7GB +[21:32:17] step 2600/4768 loss 3.1433 lr 6.00e-04 gnorm 0.12 50,370 tok/s peakVRAM 13.7GB +[21:35:46] step 2620/4768 loss 3.1518 lr 6.00e-04 gnorm 0.13 50,368 tok/s peakVRAM 13.7GB +[21:39:14] step 2640/4768 loss 3.1528 lr 6.00e-04 gnorm 0.15 50,373 tok/s peakVRAM 13.7GB +[21:42:42] step 2660/4768 loss 3.1508 lr 6.00e-04 gnorm 0.14 50,368 tok/s peakVRAM 13.7GB +[21:46:10] step 2680/4768 loss 3.1459 lr 6.00e-04 gnorm 0.13 50,369 tok/s peakVRAM 13.7GB +[21:49:38] step 2700/4768 loss 3.1484 lr 6.00e-04 gnorm 0.14 50,368 tok/s peakVRAM 13.7GB +[21:53:06] step 2720/4768 loss 3.1315 lr 6.00e-04 gnorm 0.15 50,371 tok/s peakVRAM 13.7GB +[21:56:35] step 2740/4768 loss 3.1328 lr 6.00e-04 gnorm 0.14 50,371 tok/s peakVRAM 13.7GB +[22:00:03] step 2760/4768 loss 3.1356 lr 6.00e-04 gnorm 0.14 50,367 tok/s peakVRAM 13.7GB +[22:03:31] step 2780/4768 loss 3.1263 lr 6.00e-04 gnorm 0.14 50,365 tok/s peakVRAM 13.7GB +[22:06:59] step 2800/4768 loss 3.1220 lr 6.00e-04 gnorm 0.15 50,367 tok/s peakVRAM 13.7GB +[22:10:27] step 2820/4768 loss 3.1191 lr 6.00e-04 gnorm 0.14 50,368 tok/s peakVRAM 13.7GB +[22:13:56] step 2840/4768 loss 3.1272 lr 6.00e-04 gnorm 0.16 50,369 tok/s peakVRAM 13.7GB +[22:17:24] step 2860/4768 loss 3.1184 lr 6.00e-04 gnorm 0.15 50,365 tok/s peakVRAM 13.7GB +[22:20:52] step 2880/4768 loss 3.1203 lr 6.00e-04 gnorm 0.16 50,371 tok/s peakVRAM 13.7GB +[22:24:20] step 2900/4768 loss 3.1217 lr 6.00e-04 gnorm 0.14 50,369 tok/s peakVRAM 13.7GB +[22:27:48] step 2920/4768 loss 3.1290 lr 6.00e-04 gnorm 0.17 50,368 tok/s peakVRAM 13.7GB +[22:31:16] step 2940/4768 loss 3.1193 lr 6.00e-04 gnorm 0.17 50,375 tok/s peakVRAM 13.7GB +[22:34:45] step 2960/4768 loss 3.1210 lr 6.00e-04 gnorm 0.18 50,374 tok/s peakVRAM 13.7GB +[22:38:13] step 2980/4768 loss 3.1083 lr 6.00e-04 gnorm 0.17 50,368 tok/s peakVRAM 13.7GB +[22:41:41] step 3000/4768 loss 3.1096 lr 6.00e-04 gnorm 0.15 50,369 tok/s peakVRAM 13.7GB +[22:45:09] step 3020/4768 loss 3.1041 lr 6.00e-04 gnorm 0.18 50,373 tok/s peakVRAM 13.7GB +[22:48:37] step 3040/4768 loss 3.1019 lr 6.00e-04 gnorm 0.16 50,369 tok/s peakVRAM 13.7GB +[22:52:05] step 3060/4768 loss 3.1055 lr 6.00e-04 gnorm 0.18 50,371 tok/s peakVRAM 13.7GB +[22:55:34] step 3080/4768 loss 3.0979 lr 6.00e-04 gnorm 0.18 50,369 tok/s peakVRAM 13.7GB +[22:59:02] step 3100/4768 loss 3.0916 lr 6.00e-04 gnorm 0.19 50,370 tok/s peakVRAM 13.7GB +[23:02:30] step 3120/4768 loss 3.1072 lr 6.00e-04 gnorm 0.20 50,365 tok/s peakVRAM 13.7GB +[23:05:58] step 3140/4768 loss 3.0934 lr 6.00e-04 gnorm 0.18 50,369 tok/s peakVRAM 13.7GB +[23:09:26] step 3160/4768 loss 3.0890 lr 6.00e-04 gnorm 0.17 50,366 tok/s peakVRAM 13.7GB +[23:12:55] step 3180/4768 loss 3.0953 lr 6.00e-04 gnorm 0.20 50,363 tok/s peakVRAM 13.7GB +[23:16:23] step 3200/4768 loss 3.0965 lr 6.00e-04 gnorm 0.21 50,369 tok/s peakVRAM 13.7GB +[23:19:51] step 3220/4768 loss 3.1058 lr 6.00e-04 gnorm 0.24 50,367 tok/s peakVRAM 13.7GB +[23:23:19] step 3240/4768 loss 3.0879 lr 6.00e-04 gnorm 0.22 50,369 tok/s peakVRAM 13.7GB +[23:26:47] step 3260/4768 loss 3.0842 lr 6.00e-04 gnorm 0.23 50,365 tok/s peakVRAM 13.7GB +[23:30:16] step 3280/4768 loss 3.0803 lr 6.00e-04 gnorm 0.20 50,360 tok/s peakVRAM 13.7GB +[23:33:44] step 3300/4768 loss 3.0907 lr 6.00e-04 gnorm 0.23 50,362 tok/s peakVRAM 13.7GB +[23:37:12] step 3320/4768 loss 3.0850 lr 6.00e-04 gnorm 0.27 50,360 tok/s peakVRAM 13.7GB +[23:40:40] step 3340/4768 loss 3.0855 lr 6.00e-04 gnorm 0.23 50,356 tok/s peakVRAM 13.7GB +[23:44:08] step 3360/4768 loss 3.0760 lr 6.00e-04 gnorm 0.23 50,359 tok/s peakVRAM 13.7GB +[23:47:37] step 3380/4768 loss 3.0862 lr 6.00e-04 gnorm 0.25 50,372 tok/s peakVRAM 13.7GB +[23:51:05] step 3400/4768 loss 3.0713 lr 6.00e-04 gnorm 0.25 50,370 tok/s peakVRAM 13.7GB +[23:54:33] step 3420/4768 loss 3.0788 lr 6.00e-04 gnorm 0.26 50,372 tok/s peakVRAM 13.7GB +[23:58:01] step 3440/4768 loss 3.0709 lr 6.00e-04 gnorm 0.23 50,365 tok/s peakVRAM 13.7GB +[00:01:29] step 3460/4768 loss 3.0802 lr 6.00e-04 gnorm 0.26 50,363 tok/s peakVRAM 13.7GB +[00:04:58] step 3480/4768 loss 3.0792 lr 6.00e-04 gnorm 0.24 50,364 tok/s peakVRAM 13.7GB +[00:08:26] step 3500/4768 loss 3.0655 lr 6.00e-04 gnorm 0.25 50,372 tok/s peakVRAM 13.7GB +[00:11:54] step 3520/4768 loss 3.0694 lr 6.00e-04 gnorm 0.24 50,369 tok/s peakVRAM 13.7GB +[00:15:22] step 3540/4768 loss 3.0713 lr 6.00e-04 gnorm 0.27 50,368 tok/s peakVRAM 13.7GB +[00:18:50] step 3560/4768 loss 3.0752 lr 6.00e-04 gnorm 0.26 50,361 tok/s peakVRAM 13.7GB +[00:22:18] step 3580/4768 loss 3.0689 lr 6.00e-04 gnorm 0.25 50,364 tok/s peakVRAM 13.7GB +[00:25:47] step 3600/4768 loss 3.0714 lr 6.00e-04 gnorm 0.30 50,363 tok/s peakVRAM 13.7GB +[00:29:15] step 3620/4768 loss 3.0629 lr 6.00e-04 gnorm 0.26 50,363 tok/s peakVRAM 13.7GB +[00:32:43] step 3640/4768 loss 3.0709 lr 6.00e-04 gnorm 0.28 50,359 tok/s peakVRAM 13.7GB +[00:36:11] step 3660/4768 loss 3.0597 lr 6.00e-04 gnorm 0.28 50,361 tok/s peakVRAM 13.7GB +[00:39:40] step 3680/4768 loss 3.0664 lr 6.00e-04 gnorm 0.33 50,361 tok/s peakVRAM 13.7GB +[00:43:08] step 3700/4768 loss 3.0636 lr 6.00e-04 gnorm 0.29 50,359 tok/s peakVRAM 13.7GB +[00:46:36] step 3720/4768 loss 3.0570 lr 6.00e-04 gnorm 0.28 50,362 tok/s peakVRAM 13.7GB +[00:50:04] step 3740/4768 loss 3.0528 lr 6.00e-04 gnorm 0.28 50,366 tok/s peakVRAM 13.7GB +[00:53:32] step 3760/4768 loss 3.0513 lr 6.00e-04 gnorm 0.27 50,365 tok/s peakVRAM 13.7GB +[00:57:01] step 3780/4768 loss 3.0523 lr 6.00e-04 gnorm 0.28 50,361 tok/s peakVRAM 13.7GB +[01:00:29] step 3800/4768 loss 3.0538 lr 6.00e-04 gnorm 0.30 50,363 tok/s peakVRAM 13.7GB +[01:03:57] step 3820/4768 loss 3.0467 lr 5.61e-04 gnorm 0.23 50,364 tok/s peakVRAM 13.7GB +[01:07:25] step 3840/4768 loss 3.0445 lr 5.13e-04 gnorm 0.27 50,369 tok/s peakVRAM 13.7GB +[01:10:53] step 3860/4768 loss 3.0375 lr 4.83e-04 gnorm 0.23 50,368 tok/s peakVRAM 13.7GB +[01:14:22] step 3880/4768 loss 3.0346 lr 4.59e-04 gnorm 0.20 50,368 tok/s peakVRAM 13.7GB +[01:17:50] step 3900/4768 loss 3.0270 lr 4.39e-04 gnorm 0.19 50,370 tok/s peakVRAM 13.7GB +[01:21:18] step 3920/4768 loss 3.0222 lr 4.21e-04 gnorm 0.23 50,370 tok/s peakVRAM 13.7GB +[01:24:46] step 3940/4768 loss 3.0140 lr 4.05e-04 gnorm 0.20 50,372 tok/s peakVRAM 13.7GB +[01:28:14] step 3960/4768 loss 3.0142 lr 3.89e-04 gnorm 0.15 50,372 tok/s peakVRAM 13.7GB +[01:31:42] step 3980/4768 loss 3.0063 lr 3.75e-04 gnorm 0.15 50,378 tok/s peakVRAM 13.7GB +[01:35:10] step 4000/4768 loss 3.0050 lr 3.62e-04 gnorm 0.16 50,379 tok/s peakVRAM 13.7GB +[01:38:39] step 4020/4768 loss 3.0023 lr 3.50e-04 gnorm 0.16 50,377 tok/s peakVRAM 13.7GB +[01:42:07] step 4040/4768 loss 2.9931 lr 3.38e-04 gnorm 0.14 50,372 tok/s peakVRAM 13.7GB +[01:45:35] step 4060/4768 loss 2.9928 lr 3.26e-04 gnorm 0.15 50,371 tok/s peakVRAM 13.7GB +[01:49:03] step 4080/4768 loss 2.9841 lr 3.15e-04 gnorm 0.13 50,367 tok/s peakVRAM 13.7GB +[01:52:31] step 4100/4768 loss 2.9894 lr 3.05e-04 gnorm 0.15 50,373 tok/s peakVRAM 13.7GB +[01:55:59] step 4120/4768 loss 2.9777 lr 2.95e-04 gnorm 0.14 50,378 tok/s peakVRAM 13.7GB +[01:59:28] step 4140/4768 loss 2.9702 lr 2.85e-04 gnorm 0.14 50,375 tok/s peakVRAM 13.7GB +[02:02:56] step 4160/4768 loss 2.9881 lr 2.75e-04 gnorm 0.13 50,375 tok/s peakVRAM 13.7GB +[02:06:24] step 4180/4768 loss 2.9810 lr 2.66e-04 gnorm 0.13 50,370 tok/s peakVRAM 13.7GB +[02:09:52] step 4200/4768 loss 2.9741 lr 2.57e-04 gnorm 0.13 50,372 tok/s peakVRAM 13.7GB +[02:13:20] step 4220/4768 loss 2.9675 lr 2.48e-04 gnorm 0.13 50,372 tok/s peakVRAM 13.7GB +[02:16:48] step 4240/4768 loss 2.9741 lr 2.40e-04 gnorm 0.13 50,366 tok/s peakVRAM 13.7GB +[02:20:17] step 4260/4768 loss 2.9662 lr 2.31e-04 gnorm 0.12 50,363 tok/s peakVRAM 13.7GB +[02:23:45] step 4280/4768 loss 2.9653 lr 2.23e-04 gnorm 0.13 50,366 tok/s peakVRAM 13.7GB +[02:27:13] step 4300/4768 loss 2.9630 lr 2.15e-04 gnorm 0.13 50,366 tok/s peakVRAM 13.7GB +[02:30:41] step 4320/4768 loss 2.9580 lr 2.07e-04 gnorm 0.12 50,367 tok/s peakVRAM 13.7GB +[02:34:09] step 4340/4768 loss 2.9571 lr 1.99e-04 gnorm 0.13 50,362 tok/s peakVRAM 13.7GB +[02:37:38] step 4360/4768 loss 2.9512 lr 1.92e-04 gnorm 0.13 50,366 tok/s peakVRAM 13.7GB +[02:41:06] step 4380/4768 loss 2.9582 lr 1.84e-04 gnorm 0.11 50,367 tok/s peakVRAM 13.7GB +[02:44:34] step 4400/4768 loss 2.9549 lr 1.77e-04 gnorm 0.11 50,368 tok/s peakVRAM 13.7GB +[02:48:02] step 4420/4768 loss 2.9487 lr 1.70e-04 gnorm 0.12 50,481 tok/s peakVRAM 13.7GB +[02:51:29] step 4440/4768 loss 2.9545 lr 1.63e-04 gnorm 0.12 50,542 tok/s peakVRAM 13.7GB +[02:54:57] step 4460/4768 loss 2.9502 lr 1.56e-04 gnorm 0.12 50,557 tok/s peakVRAM 13.7GB +[02:58:24] step 4480/4768 loss 2.9451 lr 1.49e-04 gnorm 0.11 50,559 tok/s peakVRAM 13.7GB +[03:01:51] step 4500/4768 loss 2.9426 lr 1.42e-04 gnorm 0.11 50,565 tok/s peakVRAM 13.7GB +[03:05:19] step 4520/4768 loss 2.9441 lr 1.36e-04 gnorm 0.12 50,564 tok/s peakVRAM 13.7GB +[03:08:46] step 4540/4768 loss 2.9361 lr 1.29e-04 gnorm 0.12 50,562 tok/s peakVRAM 13.7GB +[03:12:14] step 4560/4768 loss 2.9384 lr 1.23e-04 gnorm 0.12 50,565 tok/s peakVRAM 13.7GB +[03:15:41] step 4580/4768 loss 2.9421 lr 1.16e-04 gnorm 0.11 50,562 tok/s peakVRAM 13.7GB +[03:19:08] step 4600/4768 loss 2.9408 lr 1.10e-04 gnorm 0.11 50,556 tok/s peakVRAM 13.7GB +[03:22:36] step 4620/4768 loss 2.9273 lr 1.04e-04 gnorm 0.10 50,487 tok/s peakVRAM 13.7GB +[03:26:03] step 4640/4768 loss 2.9367 lr 9.78e-05 gnorm 0.10 50,539 tok/s peakVRAM 13.7GB +[03:29:31] step 4660/4768 loss 2.9295 lr 9.18e-05 gnorm 0.11 50,554 tok/s peakVRAM 13.7GB +[03:32:58] step 4680/4768 loss 2.9272 lr 8.58e-05 gnorm 0.10 50,523 tok/s peakVRAM 13.7GB +[03:36:26] step 4700/4768 loss 2.9233 lr 7.99e-05 gnorm 0.10 50,502 tok/s peakVRAM 13.7GB +[03:39:54] step 4720/4768 loss 2.9241 lr 7.41e-05 gnorm 0.10 50,527 tok/s peakVRAM 13.7GB +[03:43:21] step 4740/4768 loss 2.9223 lr 6.83e-05 gnorm 0.10 50,522 tok/s peakVRAM 13.7GB +[03:46:49] step 4760/4768 loss 2.9311 lr 6.26e-05 gnorm 0.10 50,546 tok/s peakVRAM 13.7GB +[03:48:13] ckpt @ 4768 +[03:48:13] DONE diff --git a/K22/run/train.out b/K22/run/train.out new file mode 100644 index 0000000000000000000000000000000000000000..ab79fa1ba1aba8a847ff15062d3c70195ca8321c --- /dev/null +++ b/K22/run/train.out @@ -0,0 +1,248 @@ +[13:59:56] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[13:59:56] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[13:59:58] model GPT-ref: 180.9M param (L22 d768 h12) +[13:59:59] optimizer=muon muon_lr=0.02 (mult=33.3x) +[13:59:59] r11 pamiec: act_ckpt=True loss_chunk=1024 +[13:59:59] r13: mixer_layout=AAAAAAAAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[14:00:00] torch.compile enabled +[14:00:00] START gpt-ref: steps=4768 (od 0) lr=0.0006->6e-05 +[14:04:48] step 20/4768 loss 10.3787 lr 2.40e-05 gnorm 2.38 36,394 tok/s peakVRAM 13.7GB +[14:08:15] step 40/4768 loss 9.6271 lr 4.80e-05 gnorm 1.40 50,498 tok/s peakVRAM 13.7GB +[14:11:43] step 60/4768 loss 8.8371 lr 7.20e-05 gnorm 1.11 50,510 tok/s peakVRAM 13.7GB +[14:15:11] step 80/4768 loss 8.1619 lr 9.60e-05 gnorm 1.04 50,497 tok/s peakVRAM 13.7GB +[14:18:38] step 100/4768 loss 7.6838 lr 1.20e-04 gnorm 0.69 50,466 tok/s peakVRAM 13.7GB +[14:22:06] step 120/4768 loss 7.2235 lr 1.44e-04 gnorm 0.83 50,441 tok/s peakVRAM 13.7GB +[14:25:34] step 140/4768 loss 6.8162 lr 1.68e-04 gnorm 0.88 50,431 tok/s peakVRAM 13.7GB +[14:29:02] step 160/4768 loss 6.4518 lr 1.92e-04 gnorm 0.96 50,418 tok/s peakVRAM 13.7GB +[14:32:30] step 180/4768 loss 6.1645 lr 2.16e-04 gnorm 0.75 50,412 tok/s peakVRAM 13.7GB +[14:35:58] step 200/4768 loss 5.9285 lr 2.40e-04 gnorm 0.78 50,411 tok/s peakVRAM 13.7GB +[14:39:26] step 220/4768 loss 5.7298 lr 2.64e-04 gnorm 0.55 50,409 tok/s peakVRAM 13.7GB +[14:42:54] step 240/4768 loss 5.5502 lr 2.88e-04 gnorm 0.49 50,408 tok/s peakVRAM 13.7GB +[14:46:22] step 260/4768 loss 5.3819 lr 3.12e-04 gnorm 0.59 50,407 tok/s peakVRAM 13.7GB +[14:49:50] step 280/4768 loss 5.2249 lr 3.36e-04 gnorm 0.58 50,410 tok/s peakVRAM 13.7GB +[14:53:18] step 300/4768 loss 5.0776 lr 3.60e-04 gnorm 0.52 50,409 tok/s peakVRAM 13.7GB +[14:56:46] step 320/4768 loss 4.9310 lr 3.84e-04 gnorm 0.63 50,410 tok/s peakVRAM 13.7GB +[15:00:14] step 340/4768 loss 4.7736 lr 4.08e-04 gnorm 0.59 50,408 tok/s peakVRAM 13.7GB +[15:03:42] step 360/4768 loss 4.6485 lr 4.32e-04 gnorm 0.58 50,401 tok/s peakVRAM 13.7GB +[15:07:10] step 380/4768 loss 4.5306 lr 4.56e-04 gnorm 0.55 50,404 tok/s peakVRAM 13.7GB +[15:10:38] step 400/4768 loss 4.4408 lr 4.80e-04 gnorm 0.48 50,402 tok/s peakVRAM 13.7GB +[15:14:06] step 420/4768 loss 4.3643 lr 5.04e-04 gnorm 0.43 50,404 tok/s peakVRAM 13.7GB +[15:17:34] step 440/4768 loss 4.2946 lr 5.28e-04 gnorm 0.50 50,403 tok/s peakVRAM 13.7GB +[15:21:02] step 460/4768 loss 4.2252 lr 5.52e-04 gnorm 0.44 50,392 tok/s peakVRAM 13.7GB +[15:24:31] step 480/4768 loss 4.1773 lr 5.76e-04 gnorm 0.43 50,388 tok/s peakVRAM 13.7GB +[15:27:59] step 500/4768 loss 4.1259 lr 6.00e-04 gnorm 0.40 50,392 tok/s peakVRAM 13.7GB +[15:31:27] step 520/4768 loss 4.0899 lr 6.00e-04 gnorm 0.33 50,395 tok/s peakVRAM 13.7GB +[15:34:55] step 540/4768 loss 4.0379 lr 6.00e-04 gnorm 0.25 50,396 tok/s peakVRAM 13.7GB +[15:38:23] step 560/4768 loss 3.9871 lr 6.00e-04 gnorm 0.33 50,389 tok/s peakVRAM 13.7GB +[15:41:51] step 580/4768 loss 3.9425 lr 6.00e-04 gnorm 0.29 50,387 tok/s peakVRAM 13.7GB +[15:45:19] step 600/4768 loss 3.8993 lr 6.00e-04 gnorm 0.26 50,382 tok/s peakVRAM 13.7GB +[15:48:47] step 620/4768 loss 3.8736 lr 6.00e-04 gnorm 0.25 50,384 tok/s peakVRAM 13.7GB +[15:52:15] step 640/4768 loss 3.8464 lr 6.00e-04 gnorm 0.28 50,384 tok/s peakVRAM 13.7GB +[15:55:44] step 660/4768 loss 3.8137 lr 6.00e-04 gnorm 0.29 50,378 tok/s peakVRAM 13.7GB +[15:59:12] step 680/4768 loss 3.7883 lr 6.00e-04 gnorm 0.22 50,379 tok/s peakVRAM 13.7GB +[16:02:40] step 700/4768 loss 3.7559 lr 6.00e-04 gnorm 0.25 50,379 tok/s peakVRAM 13.7GB +[16:06:08] step 720/4768 loss 3.7238 lr 6.00e-04 gnorm 0.25 50,377 tok/s peakVRAM 13.7GB +[16:09:36] step 740/4768 loss 3.7190 lr 6.00e-04 gnorm 0.28 50,374 tok/s peakVRAM 13.7GB +[16:13:04] step 760/4768 loss 3.6942 lr 6.00e-04 gnorm 0.22 50,379 tok/s peakVRAM 13.7GB +[16:16:32] step 780/4768 loss 3.6677 lr 6.00e-04 gnorm 0.22 50,381 tok/s peakVRAM 13.7GB +[16:20:01] step 800/4768 loss 3.6530 lr 6.00e-04 gnorm 0.20 50,378 tok/s peakVRAM 13.7GB +[16:23:29] step 820/4768 loss 3.6361 lr 6.00e-04 gnorm 0.21 50,380 tok/s peakVRAM 13.7GB +[16:26:57] step 840/4768 loss 3.6213 lr 6.00e-04 gnorm 0.21 50,379 tok/s peakVRAM 13.7GB +[16:30:25] step 860/4768 loss 3.5994 lr 6.00e-04 gnorm 0.18 50,383 tok/s peakVRAM 13.7GB +[16:33:53] step 880/4768 loss 3.5940 lr 6.00e-04 gnorm 0.21 50,370 tok/s peakVRAM 13.7GB +[16:37:21] step 900/4768 loss 3.5738 lr 6.00e-04 gnorm 0.18 50,372 tok/s peakVRAM 13.7GB +[16:40:49] step 920/4768 loss 3.5661 lr 6.00e-04 gnorm 0.19 50,372 tok/s peakVRAM 13.7GB +[16:44:18] step 940/4768 loss 3.5609 lr 6.00e-04 gnorm 0.18 50,369 tok/s peakVRAM 13.7GB +[16:47:46] step 960/4768 loss 3.5345 lr 6.00e-04 gnorm 0.17 50,374 tok/s peakVRAM 13.7GB +[16:51:14] step 980/4768 loss 3.5308 lr 6.00e-04 gnorm 0.17 50,370 tok/s peakVRAM 13.7GB +[16:54:42] step 1000/4768 loss 3.5204 lr 6.00e-04 gnorm 0.17 50,369 tok/s peakVRAM 13.7GB +[16:58:10] step 1020/4768 loss 3.5146 lr 6.00e-04 gnorm 0.19 50,360 tok/s peakVRAM 13.7GB +[17:01:39] step 1040/4768 loss 3.4869 lr 6.00e-04 gnorm 0.18 50,364 tok/s peakVRAM 13.7GB +[17:05:07] step 1060/4768 loss 3.4986 lr 6.00e-04 gnorm 0.18 50,368 tok/s peakVRAM 13.7GB +[17:08:35] step 1080/4768 loss 3.4884 lr 6.00e-04 gnorm 0.16 50,372 tok/s peakVRAM 13.7GB +[17:12:03] step 1100/4768 loss 3.4719 lr 6.00e-04 gnorm 0.21 50,367 tok/s peakVRAM 13.7GB +[17:15:31] step 1120/4768 loss 3.4548 lr 6.00e-04 gnorm 0.18 50,363 tok/s peakVRAM 13.7GB +[17:18:59] step 1140/4768 loss 3.4481 lr 6.00e-04 gnorm 0.16 50,367 tok/s peakVRAM 13.7GB +[17:22:28] step 1160/4768 loss 3.4420 lr 6.00e-04 gnorm 0.17 50,367 tok/s peakVRAM 13.7GB +[17:25:56] step 1180/4768 loss 3.4437 lr 6.00e-04 gnorm 0.16 50,366 tok/s peakVRAM 13.7GB +[17:29:24] step 1200/4768 loss 3.4253 lr 6.00e-04 gnorm 0.19 50,364 tok/s peakVRAM 13.7GB +[17:32:52] step 1220/4768 loss 3.4227 lr 6.00e-04 gnorm 0.14 50,357 tok/s peakVRAM 13.7GB +[17:36:20] step 1240/4768 loss 3.4135 lr 6.00e-04 gnorm 0.15 50,365 tok/s peakVRAM 13.7GB +[17:39:49] step 1260/4768 loss 3.3992 lr 6.00e-04 gnorm 0.15 50,366 tok/s peakVRAM 13.7GB +[17:43:17] step 1280/4768 loss 3.4017 lr 6.00e-04 gnorm 0.15 50,361 tok/s peakVRAM 13.7GB +[17:46:45] step 1300/4768 loss 3.3952 lr 6.00e-04 gnorm 0.16 50,359 tok/s peakVRAM 13.7GB +[17:50:13] step 1320/4768 loss 3.3867 lr 6.00e-04 gnorm 0.16 50,362 tok/s peakVRAM 13.7GB +[17:53:42] step 1340/4768 loss 3.3866 lr 6.00e-04 gnorm 0.15 50,360 tok/s peakVRAM 13.7GB +[17:57:10] step 1360/4768 loss 3.3772 lr 6.00e-04 gnorm 0.13 50,360 tok/s peakVRAM 13.7GB +[18:00:38] step 1380/4768 loss 3.3684 lr 6.00e-04 gnorm 0.13 50,363 tok/s peakVRAM 13.7GB +[18:04:06] step 1400/4768 loss 3.3574 lr 6.00e-04 gnorm 0.14 50,367 tok/s peakVRAM 13.7GB +[18:07:34] step 1420/4768 loss 3.3459 lr 6.00e-04 gnorm 0.16 50,368 tok/s peakVRAM 13.7GB +[18:11:02] step 1440/4768 loss 3.3518 lr 6.00e-04 gnorm 0.16 50,366 tok/s peakVRAM 13.7GB +[18:14:31] step 1460/4768 loss 3.3493 lr 6.00e-04 gnorm 0.13 50,363 tok/s peakVRAM 13.7GB +[18:17:59] step 1480/4768 loss 3.3528 lr 6.00e-04 gnorm 0.13 50,367 tok/s peakVRAM 13.7GB +[18:21:27] step 1500/4768 loss 3.3260 lr 6.00e-04 gnorm 0.16 50,361 tok/s peakVRAM 13.7GB +[18:24:55] step 1520/4768 loss 3.3333 lr 6.00e-04 gnorm 0.14 50,362 tok/s peakVRAM 13.7GB +[18:28:23] step 1540/4768 loss 3.3187 lr 6.00e-04 gnorm 0.15 50,362 tok/s peakVRAM 13.7GB +[18:31:52] step 1560/4768 loss 3.3264 lr 6.00e-04 gnorm 0.16 50,363 tok/s peakVRAM 13.7GB +[18:35:20] step 1580/4768 loss 3.3143 lr 6.00e-04 gnorm 0.12 50,366 tok/s peakVRAM 13.7GB +[18:38:48] step 1600/4768 loss 3.3036 lr 6.00e-04 gnorm 0.15 50,366 tok/s peakVRAM 13.7GB +[18:42:16] step 1620/4768 loss 3.3041 lr 6.00e-04 gnorm 0.14 50,364 tok/s peakVRAM 13.7GB +[18:45:44] step 1640/4768 loss 3.3000 lr 6.00e-04 gnorm 0.14 50,371 tok/s peakVRAM 13.7GB +[18:49:13] step 1660/4768 loss 3.3013 lr 6.00e-04 gnorm 0.13 50,376 tok/s peakVRAM 13.7GB +[18:52:41] step 1680/4768 loss 3.2893 lr 6.00e-04 gnorm 0.13 50,375 tok/s peakVRAM 13.7GB +[18:56:09] step 1700/4768 loss 3.2815 lr 6.00e-04 gnorm 0.12 50,373 tok/s peakVRAM 13.7GB +[18:59:37] step 1720/4768 loss 3.2840 lr 6.00e-04 gnorm 0.15 50,365 tok/s peakVRAM 13.7GB +[19:03:05] step 1740/4768 loss 3.2778 lr 6.00e-04 gnorm 0.13 50,366 tok/s peakVRAM 13.7GB +[19:06:34] step 1760/4768 loss 3.2767 lr 6.00e-04 gnorm 0.13 50,363 tok/s peakVRAM 13.7GB +[19:10:02] step 1780/4768 loss 3.2649 lr 6.00e-04 gnorm 0.14 50,359 tok/s peakVRAM 13.7GB +[19:13:30] step 1800/4768 loss 3.2683 lr 6.00e-04 gnorm 0.13 50,354 tok/s peakVRAM 13.7GB +[19:16:58] step 1820/4768 loss 3.2637 lr 6.00e-04 gnorm 0.12 50,363 tok/s peakVRAM 13.7GB +[19:20:26] step 1840/4768 loss 3.2567 lr 6.00e-04 gnorm 0.12 50,360 tok/s peakVRAM 13.7GB +[19:23:55] step 1860/4768 loss 3.2561 lr 6.00e-04 gnorm 0.13 50,361 tok/s peakVRAM 13.7GB +[19:27:23] step 1880/4768 loss 3.2509 lr 6.00e-04 gnorm 0.13 50,368 tok/s peakVRAM 13.7GB +[19:30:51] step 1900/4768 loss 3.2456 lr 6.00e-04 gnorm 0.12 50,367 tok/s peakVRAM 13.7GB +[19:34:19] step 1920/4768 loss 3.2454 lr 6.00e-04 gnorm 0.11 50,365 tok/s peakVRAM 13.7GB +[19:37:47] step 1940/4768 loss 3.2371 lr 6.00e-04 gnorm 0.13 50,361 tok/s peakVRAM 13.7GB +[19:41:16] step 1960/4768 loss 3.2425 lr 6.00e-04 gnorm 0.13 50,367 tok/s peakVRAM 13.7GB +[19:44:44] step 1980/4768 loss 3.2365 lr 6.00e-04 gnorm 0.12 50,369 tok/s peakVRAM 13.7GB +[19:48:12] step 2000/4768 loss 3.2297 lr 6.00e-04 gnorm 0.13 50,370 tok/s peakVRAM 13.7GB +[19:51:40] step 2020/4768 loss 3.2300 lr 6.00e-04 gnorm 0.13 50,368 tok/s peakVRAM 13.7GB +[19:55:08] step 2040/4768 loss 3.2229 lr 6.00e-04 gnorm 0.13 50,366 tok/s peakVRAM 13.7GB +[19:58:36] step 2060/4768 loss 3.2234 lr 6.00e-04 gnorm 0.12 50,367 tok/s peakVRAM 13.7GB +[20:02:05] step 2080/4768 loss 3.2119 lr 6.00e-04 gnorm 0.13 50,369 tok/s peakVRAM 13.7GB +[20:05:33] step 2100/4768 loss 3.2206 lr 6.00e-04 gnorm 0.11 50,369 tok/s peakVRAM 13.7GB +[20:09:01] step 2120/4768 loss 3.2098 lr 6.00e-04 gnorm 0.13 50,366 tok/s peakVRAM 13.7GB +[20:12:29] step 2140/4768 loss 3.2225 lr 6.00e-04 gnorm 0.13 50,370 tok/s peakVRAM 13.7GB +[20:15:57] step 2160/4768 loss 3.2009 lr 6.00e-04 gnorm 0.13 50,369 tok/s peakVRAM 13.7GB +[20:19:26] step 2180/4768 loss 3.2029 lr 6.00e-04 gnorm 0.11 50,369 tok/s peakVRAM 13.7GB +[20:22:54] step 2200/4768 loss 3.2040 lr 6.00e-04 gnorm 0.12 50,375 tok/s peakVRAM 13.7GB +[20:26:22] step 2220/4768 loss 3.1966 lr 6.00e-04 gnorm 0.15 50,370 tok/s peakVRAM 13.7GB +[20:29:50] step 2240/4768 loss 3.1929 lr 6.00e-04 gnorm 0.12 50,366 tok/s peakVRAM 13.7GB +[20:33:18] step 2260/4768 loss 3.1895 lr 6.00e-04 gnorm 0.12 50,364 tok/s peakVRAM 13.7GB +[20:36:46] step 2280/4768 loss 3.1854 lr 6.00e-04 gnorm 0.12 50,367 tok/s peakVRAM 13.7GB +[20:40:15] step 2300/4768 loss 3.1888 lr 6.00e-04 gnorm 0.12 50,365 tok/s peakVRAM 13.7GB +[20:43:43] step 2320/4768 loss 3.1856 lr 6.00e-04 gnorm 0.13 50,366 tok/s peakVRAM 13.7GB +[20:47:11] step 2340/4768 loss 3.1806 lr 6.00e-04 gnorm 0.14 50,371 tok/s peakVRAM 13.7GB +[20:50:39] step 2360/4768 loss 3.1701 lr 6.00e-04 gnorm 0.12 50,374 tok/s peakVRAM 13.7GB +[20:54:07] step 2380/4768 loss 3.1692 lr 6.00e-04 gnorm 0.13 50,367 tok/s peakVRAM 13.7GB +[20:57:36] step 2400/4768 loss 3.1698 lr 6.00e-04 gnorm 0.12 50,363 tok/s peakVRAM 13.7GB +[21:01:04] step 2420/4768 loss 3.1726 lr 6.00e-04 gnorm 0.12 50,363 tok/s peakVRAM 13.7GB +[21:04:32] step 2440/4768 loss 3.1677 lr 6.00e-04 gnorm 0.14 50,365 tok/s peakVRAM 13.7GB +[21:08:00] step 2460/4768 loss 3.1653 lr 6.00e-04 gnorm 0.15 50,361 tok/s peakVRAM 13.7GB +[21:11:28] step 2480/4768 loss 3.1631 lr 6.00e-04 gnorm 0.13 50,368 tok/s peakVRAM 13.7GB +[21:14:57] step 2500/4768 loss 3.1575 lr 6.00e-04 gnorm 0.11 50,371 tok/s peakVRAM 13.7GB +[21:18:25] step 2520/4768 loss 3.1594 lr 6.00e-04 gnorm 0.12 50,366 tok/s peakVRAM 13.7GB +[21:21:53] step 2540/4768 loss 3.1598 lr 6.00e-04 gnorm 0.13 50,370 tok/s peakVRAM 13.7GB +[21:25:21] step 2560/4768 loss 3.1564 lr 6.00e-04 gnorm 0.13 50,377 tok/s peakVRAM 13.7GB +[21:28:49] step 2580/4768 loss 3.1529 lr 6.00e-04 gnorm 0.14 50,374 tok/s peakVRAM 13.7GB +[21:32:17] step 2600/4768 loss 3.1433 lr 6.00e-04 gnorm 0.12 50,370 tok/s peakVRAM 13.7GB +[21:35:46] step 2620/4768 loss 3.1518 lr 6.00e-04 gnorm 0.13 50,368 tok/s peakVRAM 13.7GB +[21:39:14] step 2640/4768 loss 3.1528 lr 6.00e-04 gnorm 0.15 50,373 tok/s peakVRAM 13.7GB +[21:42:42] step 2660/4768 loss 3.1508 lr 6.00e-04 gnorm 0.14 50,368 tok/s peakVRAM 13.7GB +[21:46:10] step 2680/4768 loss 3.1459 lr 6.00e-04 gnorm 0.13 50,369 tok/s peakVRAM 13.7GB +[21:49:38] step 2700/4768 loss 3.1484 lr 6.00e-04 gnorm 0.14 50,368 tok/s peakVRAM 13.7GB +[21:53:06] step 2720/4768 loss 3.1315 lr 6.00e-04 gnorm 0.15 50,371 tok/s peakVRAM 13.7GB +[21:56:35] step 2740/4768 loss 3.1328 lr 6.00e-04 gnorm 0.14 50,371 tok/s peakVRAM 13.7GB +[22:00:03] step 2760/4768 loss 3.1356 lr 6.00e-04 gnorm 0.14 50,367 tok/s peakVRAM 13.7GB +[22:03:31] step 2780/4768 loss 3.1263 lr 6.00e-04 gnorm 0.14 50,365 tok/s peakVRAM 13.7GB +[22:06:59] step 2800/4768 loss 3.1220 lr 6.00e-04 gnorm 0.15 50,367 tok/s peakVRAM 13.7GB +[22:10:27] step 2820/4768 loss 3.1191 lr 6.00e-04 gnorm 0.14 50,368 tok/s peakVRAM 13.7GB +[22:13:56] step 2840/4768 loss 3.1272 lr 6.00e-04 gnorm 0.16 50,369 tok/s peakVRAM 13.7GB +[22:17:24] step 2860/4768 loss 3.1184 lr 6.00e-04 gnorm 0.15 50,365 tok/s peakVRAM 13.7GB +[22:20:52] step 2880/4768 loss 3.1203 lr 6.00e-04 gnorm 0.16 50,371 tok/s peakVRAM 13.7GB +[22:24:20] step 2900/4768 loss 3.1217 lr 6.00e-04 gnorm 0.14 50,369 tok/s peakVRAM 13.7GB +[22:27:48] step 2920/4768 loss 3.1290 lr 6.00e-04 gnorm 0.17 50,368 tok/s peakVRAM 13.7GB +[22:31:16] step 2940/4768 loss 3.1193 lr 6.00e-04 gnorm 0.17 50,375 tok/s peakVRAM 13.7GB +[22:34:45] step 2960/4768 loss 3.1210 lr 6.00e-04 gnorm 0.18 50,374 tok/s peakVRAM 13.7GB +[22:38:13] step 2980/4768 loss 3.1083 lr 6.00e-04 gnorm 0.17 50,368 tok/s peakVRAM 13.7GB +[22:41:41] step 3000/4768 loss 3.1096 lr 6.00e-04 gnorm 0.15 50,369 tok/s peakVRAM 13.7GB +[22:45:09] step 3020/4768 loss 3.1041 lr 6.00e-04 gnorm 0.18 50,373 tok/s peakVRAM 13.7GB +[22:48:37] step 3040/4768 loss 3.1019 lr 6.00e-04 gnorm 0.16 50,369 tok/s peakVRAM 13.7GB +[22:52:05] step 3060/4768 loss 3.1055 lr 6.00e-04 gnorm 0.18 50,371 tok/s peakVRAM 13.7GB +[22:55:34] step 3080/4768 loss 3.0979 lr 6.00e-04 gnorm 0.18 50,369 tok/s peakVRAM 13.7GB +[22:59:02] step 3100/4768 loss 3.0916 lr 6.00e-04 gnorm 0.19 50,370 tok/s peakVRAM 13.7GB +[23:02:30] step 3120/4768 loss 3.1072 lr 6.00e-04 gnorm 0.20 50,365 tok/s peakVRAM 13.7GB +[23:05:58] step 3140/4768 loss 3.0934 lr 6.00e-04 gnorm 0.18 50,369 tok/s peakVRAM 13.7GB +[23:09:26] step 3160/4768 loss 3.0890 lr 6.00e-04 gnorm 0.17 50,366 tok/s peakVRAM 13.7GB +[23:12:55] step 3180/4768 loss 3.0953 lr 6.00e-04 gnorm 0.20 50,363 tok/s peakVRAM 13.7GB +[23:16:23] step 3200/4768 loss 3.0965 lr 6.00e-04 gnorm 0.21 50,369 tok/s peakVRAM 13.7GB +[23:19:51] step 3220/4768 loss 3.1058 lr 6.00e-04 gnorm 0.24 50,367 tok/s peakVRAM 13.7GB +[23:23:19] step 3240/4768 loss 3.0879 lr 6.00e-04 gnorm 0.22 50,369 tok/s peakVRAM 13.7GB +[23:26:47] step 3260/4768 loss 3.0842 lr 6.00e-04 gnorm 0.23 50,365 tok/s peakVRAM 13.7GB +[23:30:16] step 3280/4768 loss 3.0803 lr 6.00e-04 gnorm 0.20 50,360 tok/s peakVRAM 13.7GB +[23:33:44] step 3300/4768 loss 3.0907 lr 6.00e-04 gnorm 0.23 50,362 tok/s peakVRAM 13.7GB +[23:37:12] step 3320/4768 loss 3.0850 lr 6.00e-04 gnorm 0.27 50,360 tok/s peakVRAM 13.7GB +[23:40:40] step 3340/4768 loss 3.0855 lr 6.00e-04 gnorm 0.23 50,356 tok/s peakVRAM 13.7GB +[23:44:08] step 3360/4768 loss 3.0760 lr 6.00e-04 gnorm 0.23 50,359 tok/s peakVRAM 13.7GB +[23:47:37] step 3380/4768 loss 3.0862 lr 6.00e-04 gnorm 0.25 50,372 tok/s peakVRAM 13.7GB +[23:51:05] step 3400/4768 loss 3.0713 lr 6.00e-04 gnorm 0.25 50,370 tok/s peakVRAM 13.7GB +[23:54:33] step 3420/4768 loss 3.0788 lr 6.00e-04 gnorm 0.26 50,372 tok/s peakVRAM 13.7GB +[23:58:01] step 3440/4768 loss 3.0709 lr 6.00e-04 gnorm 0.23 50,365 tok/s peakVRAM 13.7GB +[00:01:29] step 3460/4768 loss 3.0802 lr 6.00e-04 gnorm 0.26 50,363 tok/s peakVRAM 13.7GB +[00:04:58] step 3480/4768 loss 3.0792 lr 6.00e-04 gnorm 0.24 50,364 tok/s peakVRAM 13.7GB +[00:08:26] step 3500/4768 loss 3.0655 lr 6.00e-04 gnorm 0.25 50,372 tok/s peakVRAM 13.7GB +[00:11:54] step 3520/4768 loss 3.0694 lr 6.00e-04 gnorm 0.24 50,369 tok/s peakVRAM 13.7GB +[00:15:22] step 3540/4768 loss 3.0713 lr 6.00e-04 gnorm 0.27 50,368 tok/s peakVRAM 13.7GB +[00:18:50] step 3560/4768 loss 3.0752 lr 6.00e-04 gnorm 0.26 50,361 tok/s peakVRAM 13.7GB +[00:22:18] step 3580/4768 loss 3.0689 lr 6.00e-04 gnorm 0.25 50,364 tok/s peakVRAM 13.7GB +[00:25:47] step 3600/4768 loss 3.0714 lr 6.00e-04 gnorm 0.30 50,363 tok/s peakVRAM 13.7GB +[00:29:15] step 3620/4768 loss 3.0629 lr 6.00e-04 gnorm 0.26 50,363 tok/s peakVRAM 13.7GB +[00:32:43] step 3640/4768 loss 3.0709 lr 6.00e-04 gnorm 0.28 50,359 tok/s peakVRAM 13.7GB +[00:36:11] step 3660/4768 loss 3.0597 lr 6.00e-04 gnorm 0.28 50,361 tok/s peakVRAM 13.7GB +[00:39:40] step 3680/4768 loss 3.0664 lr 6.00e-04 gnorm 0.33 50,361 tok/s peakVRAM 13.7GB +[00:43:08] step 3700/4768 loss 3.0636 lr 6.00e-04 gnorm 0.29 50,359 tok/s peakVRAM 13.7GB +[00:46:36] step 3720/4768 loss 3.0570 lr 6.00e-04 gnorm 0.28 50,362 tok/s peakVRAM 13.7GB +[00:50:04] step 3740/4768 loss 3.0528 lr 6.00e-04 gnorm 0.28 50,366 tok/s peakVRAM 13.7GB +[00:53:32] step 3760/4768 loss 3.0513 lr 6.00e-04 gnorm 0.27 50,365 tok/s peakVRAM 13.7GB +[00:57:01] step 3780/4768 loss 3.0523 lr 6.00e-04 gnorm 0.28 50,361 tok/s peakVRAM 13.7GB +[01:00:29] step 3800/4768 loss 3.0538 lr 6.00e-04 gnorm 0.30 50,363 tok/s peakVRAM 13.7GB +[01:03:57] step 3820/4768 loss 3.0467 lr 5.61e-04 gnorm 0.23 50,364 tok/s peakVRAM 13.7GB +[01:07:25] step 3840/4768 loss 3.0445 lr 5.13e-04 gnorm 0.27 50,369 tok/s peakVRAM 13.7GB +[01:10:53] step 3860/4768 loss 3.0375 lr 4.83e-04 gnorm 0.23 50,368 tok/s peakVRAM 13.7GB +[01:14:22] step 3880/4768 loss 3.0346 lr 4.59e-04 gnorm 0.20 50,368 tok/s peakVRAM 13.7GB +[01:17:50] step 3900/4768 loss 3.0270 lr 4.39e-04 gnorm 0.19 50,370 tok/s peakVRAM 13.7GB +[01:21:18] step 3920/4768 loss 3.0222 lr 4.21e-04 gnorm 0.23 50,370 tok/s peakVRAM 13.7GB +[01:24:46] step 3940/4768 loss 3.0140 lr 4.05e-04 gnorm 0.20 50,372 tok/s peakVRAM 13.7GB +[01:28:14] step 3960/4768 loss 3.0142 lr 3.89e-04 gnorm 0.15 50,372 tok/s peakVRAM 13.7GB +[01:31:42] step 3980/4768 loss 3.0063 lr 3.75e-04 gnorm 0.15 50,378 tok/s peakVRAM 13.7GB +[01:35:10] step 4000/4768 loss 3.0050 lr 3.62e-04 gnorm 0.16 50,379 tok/s peakVRAM 13.7GB +[01:38:39] step 4020/4768 loss 3.0023 lr 3.50e-04 gnorm 0.16 50,377 tok/s peakVRAM 13.7GB +[01:42:07] step 4040/4768 loss 2.9931 lr 3.38e-04 gnorm 0.14 50,372 tok/s peakVRAM 13.7GB +[01:45:35] step 4060/4768 loss 2.9928 lr 3.26e-04 gnorm 0.15 50,371 tok/s peakVRAM 13.7GB +[01:49:03] step 4080/4768 loss 2.9841 lr 3.15e-04 gnorm 0.13 50,367 tok/s peakVRAM 13.7GB +[01:52:31] step 4100/4768 loss 2.9894 lr 3.05e-04 gnorm 0.15 50,373 tok/s peakVRAM 13.7GB +[01:55:59] step 4120/4768 loss 2.9777 lr 2.95e-04 gnorm 0.14 50,378 tok/s peakVRAM 13.7GB +[01:59:28] step 4140/4768 loss 2.9702 lr 2.85e-04 gnorm 0.14 50,375 tok/s peakVRAM 13.7GB +[02:02:56] step 4160/4768 loss 2.9881 lr 2.75e-04 gnorm 0.13 50,375 tok/s peakVRAM 13.7GB +[02:06:24] step 4180/4768 loss 2.9810 lr 2.66e-04 gnorm 0.13 50,370 tok/s peakVRAM 13.7GB +[02:09:52] step 4200/4768 loss 2.9741 lr 2.57e-04 gnorm 0.13 50,372 tok/s peakVRAM 13.7GB +[02:13:20] step 4220/4768 loss 2.9675 lr 2.48e-04 gnorm 0.13 50,372 tok/s peakVRAM 13.7GB +[02:16:48] step 4240/4768 loss 2.9741 lr 2.40e-04 gnorm 0.13 50,366 tok/s peakVRAM 13.7GB +[02:20:17] step 4260/4768 loss 2.9662 lr 2.31e-04 gnorm 0.12 50,363 tok/s peakVRAM 13.7GB +[02:23:45] step 4280/4768 loss 2.9653 lr 2.23e-04 gnorm 0.13 50,366 tok/s peakVRAM 13.7GB +[02:27:13] step 4300/4768 loss 2.9630 lr 2.15e-04 gnorm 0.13 50,366 tok/s peakVRAM 13.7GB +[02:30:41] step 4320/4768 loss 2.9580 lr 2.07e-04 gnorm 0.12 50,367 tok/s peakVRAM 13.7GB +[02:34:09] step 4340/4768 loss 2.9571 lr 1.99e-04 gnorm 0.13 50,362 tok/s peakVRAM 13.7GB +[02:37:38] step 4360/4768 loss 2.9512 lr 1.92e-04 gnorm 0.13 50,366 tok/s peakVRAM 13.7GB +[02:41:06] step 4380/4768 loss 2.9582 lr 1.84e-04 gnorm 0.11 50,367 tok/s peakVRAM 13.7GB +[02:44:34] step 4400/4768 loss 2.9549 lr 1.77e-04 gnorm 0.11 50,368 tok/s peakVRAM 13.7GB +[02:48:02] step 4420/4768 loss 2.9487 lr 1.70e-04 gnorm 0.12 50,481 tok/s peakVRAM 13.7GB +[02:51:29] step 4440/4768 loss 2.9545 lr 1.63e-04 gnorm 0.12 50,542 tok/s peakVRAM 13.7GB +[02:54:57] step 4460/4768 loss 2.9502 lr 1.56e-04 gnorm 0.12 50,557 tok/s peakVRAM 13.7GB +[02:58:24] step 4480/4768 loss 2.9451 lr 1.49e-04 gnorm 0.11 50,559 tok/s peakVRAM 13.7GB +[03:01:51] step 4500/4768 loss 2.9426 lr 1.42e-04 gnorm 0.11 50,565 tok/s peakVRAM 13.7GB +[03:05:19] step 4520/4768 loss 2.9441 lr 1.36e-04 gnorm 0.12 50,564 tok/s peakVRAM 13.7GB +[03:08:46] step 4540/4768 loss 2.9361 lr 1.29e-04 gnorm 0.12 50,562 tok/s peakVRAM 13.7GB +[03:12:14] step 4560/4768 loss 2.9384 lr 1.23e-04 gnorm 0.12 50,565 tok/s peakVRAM 13.7GB +[03:15:41] step 4580/4768 loss 2.9421 lr 1.16e-04 gnorm 0.11 50,562 tok/s peakVRAM 13.7GB +[03:19:08] step 4600/4768 loss 2.9408 lr 1.10e-04 gnorm 0.11 50,556 tok/s peakVRAM 13.7GB +[03:22:36] step 4620/4768 loss 2.9273 lr 1.04e-04 gnorm 0.10 50,487 tok/s peakVRAM 13.7GB +[03:26:03] step 4640/4768 loss 2.9367 lr 9.78e-05 gnorm 0.10 50,539 tok/s peakVRAM 13.7GB +[03:29:31] step 4660/4768 loss 2.9295 lr 9.18e-05 gnorm 0.11 50,554 tok/s peakVRAM 13.7GB +[03:32:58] step 4680/4768 loss 2.9272 lr 8.58e-05 gnorm 0.10 50,523 tok/s peakVRAM 13.7GB +[03:36:26] step 4700/4768 loss 2.9233 lr 7.99e-05 gnorm 0.10 50,502 tok/s peakVRAM 13.7GB +[03:39:54] step 4720/4768 loss 2.9241 lr 7.41e-05 gnorm 0.10 50,527 tok/s peakVRAM 13.7GB +[03:43:21] step 4740/4768 loss 2.9223 lr 6.83e-05 gnorm 0.10 50,522 tok/s peakVRAM 13.7GB +[03:46:49] step 4760/4768 loss 2.9311 lr 6.26e-05 gnorm 0.10 50,546 tok/s peakVRAM 13.7GB +[03:48:13] ckpt @ 4768 +[03:48:13] DONE diff --git a/K22/run/val_en.json b/K22/run/val_en.json new file mode 100644 index 0000000000000000000000000000000000000000..fcdfe99990630e00eb29df9beef1171e0a0b14c8 --- /dev/null +++ b/K22/run/val_en.json @@ -0,0 +1 @@ +{"schema": "val_bpb_v0", "ckpt_sha256": "0840bf3078d0f84fc20e68337cc24e8a9e4bba145f7ea0d27541792c1dd7f189", "step": 4768, "val_sha256": "046bc03b426b75f0e84db7cdf3965a9653d94e722b81b0d03fd0cb92884cf0c5", "tokenizer_sha256": "0640d3bd3674d7a6e59c540d945a2ad275a12e1bcaeb6007c37a90751f9815dd", "script_sha256": "ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1", "trainer_sha256": "db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b", "window": 4096, "n_windows": 547, "tokens": 2240512, "bytes": 9799872, "sum_nat": 6818977.61328125, "bpb": 1.0038605796701956, "ce_token": 3.0434907794652517, "P": 180916757, "dtype": "fp32, tf32 off, bez autocast", "torch": "2.6.0+cu124", "device": "NVIDIA GeForce RTX 4090"} \ No newline at end of file diff --git a/K22/run/val_en.out b/K22/run/val_en.out new file mode 100644 index 0000000000000000000000000000000000000000..668490a4bfde1ab16bb3b1c30b6634572ec31faf --- /dev/null +++ b/K22/run/val_en.out @@ -0,0 +1 @@ +VAL-BPB-OK step 4768 bpb 1.00386 ce 3.04349 bajty 9799872 P 180916757 -> /home/ubuntu/kolektyw/dowody400m/wyniki/K22/val_en.json diff --git a/K22/run/val_pl.json b/K22/run/val_pl.json new file mode 100644 index 0000000000000000000000000000000000000000..6473827b0bf986ca72bc6f8e7ca20a8f84315dde --- /dev/null +++ b/K22/run/val_pl.json @@ -0,0 +1 @@ +{"schema": "val_bpb_v0", "ckpt_sha256": "0840bf3078d0f84fc20e68337cc24e8a9e4bba145f7ea0d27541792c1dd7f189", "step": 4768, "val_sha256": "99e66265f47d249c72f5e76838aa23e82959b99089f720bea6e9ee2cb1f49ee1", "tokenizer_sha256": "0640d3bd3674d7a6e59c540d945a2ad275a12e1bcaeb6007c37a90751f9815dd", "script_sha256": "ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1", "trainer_sha256": "db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b", "window": 4096, "n_windows": 406, "tokens": 1662976, "bytes": 7234678, "sum_nat": 5472073.984375, "bpb": 1.0912073765598584, "ce_token": 3.290530942343726, "P": 180916757, "dtype": "fp32, tf32 off, bez autocast", "torch": "2.6.0+cu124", "device": "NVIDIA GeForce RTX 4090"} \ No newline at end of file diff --git a/K22/run/val_pl.out b/K22/run/val_pl.out new file mode 100644 index 0000000000000000000000000000000000000000..7f28e8b75f92abe8419d6ef120bc14f8d3378d70 --- /dev/null +++ b/K22/run/val_pl.out @@ -0,0 +1 @@ +VAL-BPB-OK step 4768 bpb 1.09121 ce 3.29053 bajty 7234678 P 180916757 -> /home/ubuntu/kolektyw/dowody400m/wyniki/K22/val_pl.json diff --git a/K22/run/wynik.json b/K22/run/wynik.json new file mode 100644 index 0000000000000000000000000000000000000000..87982f0c739f9479b3e2514fda7fa800acfcce1a --- /dev/null +++ b/K22/run/wynik.json @@ -0,0 +1 @@ +{"ramie": "K22", "ckpt_sha256": "0840bf3078d0f84fc20e68337cc24e8a9e4bba145f7ea0d27541792c1dd7f189", "bpb_pl": 1.0912073765598584, "bpb_en": 1.0038605796701956, "bpb_wazona_40_60": 1.0387992984260608, "val_pl_sha256": "99e66265f47d249c72f5e76838aa23e82959b99089f720bea6e9ee2cb1f49ee1", "val_en_sha256": "046bc03b426b75f0e84db7cdf3965a9653d94e722b81b0d03fd0cb92884cf0c5", "P": 180916757, "step": 4768} diff --git a/MU2/run/SHA256SUMS_kopiarka b/MU2/run/SHA256SUMS_kopiarka new file mode 100644 index 0000000000000000000000000000000000000000..724fa48f575ed6a6a9a47e9fed98a52a02eebe98 --- /dev/null +++ b/MU2/run/SHA256SUMS_kopiarka @@ -0,0 +1,4 @@ +dafb39311a0abbd298efc00372e8d574ef255c2e254dd964f99f92df7ad5116e ckpt_500.pt +d1c16074f435fed71a47e0c377bc3332344ad5bc29c4dfef86dee55c66a7f396 ckpt_1000.pt +edf8bdeae17f21b10e59fe5f56e793a3bfe92a5a28da86d1d8eaf21d8fad6602 ckpt_1500.pt +72507b5463e7716781fe783b1599cac8dd2fcb9c10823d47aada75c2bf5b7248 ckpt_2000.pt diff --git a/MU2/run/args.txt b/MU2/run/args.txt new file mode 100644 index 0000000000000000000000000000000000000000..b3bcbdc76a1ffd3c73321703eaf63fd0f4c29f35 --- /dev/null +++ b/MU2/run/args.txt @@ -0,0 +1 @@ +--data-dir /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2 --out-dir /home/ubuntu/kolektyw/dowody400m/wyniki/MU2 --run-id dowody_MU2 --vocab 32768 --dtype uint16 --block 4096 --n-layer 16 --n-embd 896 --n-head 14 --value-residual --batch 128 --grad-accum 8 --norm rmsnorm --pos rope --rope-theta 500000 --ffn swiglu --ffn-mult 2.667 --qk-norm --attn-bias 0 --optimizer muon --muon-lr 0.04 --lr 6e-4 --min-lr 6e-05 --wd 0.1 --schedule wsd --decay-frac 0.2 --warmup 500 --act-ckpt --loss-chunk 1024 --sampler random --seed 1 --data-seed 20261022 --compile --log-every 20 --eval-every 4769 --eval-iters 1 --steps 4768 --ckpt-every 500 \ No newline at end of file diff --git a/MU2/run/metrics.jsonl b/MU2/run/metrics.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..af587fd7f2f9479ad85e72d96e869698bcfbfbd3 --- /dev/null +++ b/MU2/run/metrics.jsonl @@ -0,0 +1,117 @@ +{"step": 20, "loss": 10.25530424118042, "lr": 2.3999999999999997e-05, "tok_s": 44303.59020061327} +{"step": 40, "loss": 9.296780109405518, "lr": 4.7999999999999994e-05, "tok_s": 54450.233286746974} +{"step": 60, "loss": 8.495148754119873, "lr": 7.199999999999999e-05, "tok_s": 54449.48407584481} +{"step": 80, "loss": 7.82485179901123, "lr": 9.599999999999999e-05, "tok_s": 54438.767439872885} +{"step": 100, "loss": 7.302724242210388, "lr": 0.00011999999999999999, "tok_s": 54420.73828892327} +{"step": 120, "loss": 6.796665596961975, "lr": 0.00014399999999999998, "tok_s": 54419.8315794307} +{"step": 140, "loss": 6.40500955581665, "lr": 0.000168, "tok_s": 54415.44303424002} +{"step": 160, "loss": 6.1027823209762575, "lr": 0.00019199999999999998, "tok_s": 54413.32530167493} +{"step": 180, "loss": 5.878649806976318, "lr": 0.00021599999999999996, "tok_s": 54414.32557584492} +{"step": 200, "loss": 5.675228881835937, "lr": 0.00023999999999999998, "tok_s": 54411.914760814034} +{"step": 220, "loss": 5.494605636596679, "lr": 0.00026399999999999997, "tok_s": 54411.29134027191} +{"step": 240, "loss": 5.32799596786499, "lr": 0.00028799999999999995, "tok_s": 54408.15432666468} +{"step": 260, "loss": 5.167963361740112, "lr": 0.000312, "tok_s": 54407.70774022487} +{"step": 280, "loss": 5.026251649856567, "lr": 0.000336, "tok_s": 54403.17558868095} +{"step": 300, "loss": 4.884607744216919, "lr": 0.00035999999999999997, "tok_s": 54403.6589795126} +{"step": 320, "loss": 4.747219347953797, "lr": 0.00038399999999999996, "tok_s": 54402.308561967555} +{"step": 340, "loss": 4.611021494865417, "lr": 0.000408, "tok_s": 54398.66464668593} +{"step": 360, "loss": 4.5043387651443485, "lr": 0.00043199999999999993, "tok_s": 54397.36379438711} +{"step": 380, "loss": 4.405326771736145, "lr": 0.00045599999999999997, "tok_s": 54389.69034144712} +{"step": 400, "loss": 4.3193655729293825, "lr": 0.00047999999999999996, "tok_s": 54388.95342288856} +{"step": 420, "loss": 4.254396152496338, "lr": 0.000504, "tok_s": 54392.29332507222} +{"step": 440, "loss": 4.184388732910156, "lr": 0.0005279999999999999, "tok_s": 54391.702709732206} +{"step": 460, "loss": 4.12065064907074, "lr": 0.000552, "tok_s": 54389.70399574904} +{"step": 480, "loss": 4.079915833473206, "lr": 0.0005759999999999999, "tok_s": 54388.5113889538} +{"step": 500, "loss": 4.029127311706543, "lr": 0.0006, "tok_s": 54385.7028387596} +{"step": 520, "loss": 3.99461567401886, "lr": 0.0006, "tok_s": 53917.76280203575} +{"step": 540, "loss": 3.9496883153915405, "lr": 0.0006, "tok_s": 54387.81579867178} +{"step": 560, "loss": 3.8994050145149233, "lr": 0.0006, "tok_s": 54386.183497997205} +{"step": 580, "loss": 3.8611037373542785, "lr": 0.0006, "tok_s": 54384.45109321471} +{"step": 600, "loss": 3.8205567002296448, "lr": 0.0006, "tok_s": 54386.24853257764} +{"step": 620, "loss": 3.79986629486084, "lr": 0.0006, "tok_s": 54384.0326021977} +{"step": 640, "loss": 3.775229072570801, "lr": 0.0006, "tok_s": 54381.57639218699} +{"step": 660, "loss": 3.747225058078766, "lr": 0.0006, "tok_s": 54379.4114047863} +{"step": 680, "loss": 3.7248820304870605, "lr": 0.0006, "tok_s": 54375.93439154698} +{"step": 700, "loss": 3.6958727717399595, "lr": 0.0006, "tok_s": 54382.27256256626} +{"step": 720, "loss": 3.6669984459877014, "lr": 0.0006, "tok_s": 54381.27561820643} +{"step": 740, "loss": 3.663788890838623, "lr": 0.0006, "tok_s": 54382.09194522191} +{"step": 760, "loss": 3.6385889649391174, "lr": 0.0006, "tok_s": 54382.89444411851} +{"step": 780, "loss": 3.6130353808403015, "lr": 0.0006, "tok_s": 54386.28518605501} +{"step": 800, "loss": 3.6004954099655153, "lr": 0.0006, "tok_s": 54384.68539181737} +{"step": 820, "loss": 3.5843114852905273, "lr": 0.0006, "tok_s": 54385.4770051577} +{"step": 840, "loss": 3.5712699770927427, "lr": 0.0006, "tok_s": 54381.674364666964} +{"step": 860, "loss": 3.5506399631500245, "lr": 0.0006, "tok_s": 54383.550703277906} +{"step": 880, "loss": 3.545134401321411, "lr": 0.0006, "tok_s": 54385.182509659986} +{"step": 900, "loss": 3.5280290246009827, "lr": 0.0006, "tok_s": 54387.28069994241} +{"step": 920, "loss": 3.5207173466682433, "lr": 0.0006, "tok_s": 54382.421710629926} +{"step": 940, "loss": 3.51641960144043, "lr": 0.0006, "tok_s": 54383.32865337053} +{"step": 960, "loss": 3.491331160068512, "lr": 0.0006, "tok_s": 54381.12076091297} +{"step": 980, "loss": 3.488969933986664, "lr": 0.0006, "tok_s": 54379.83937250632} +{"step": 1000, "loss": 3.4787667393684387, "lr": 0.0006, "tok_s": 54379.45826910843} +{"step": 1020, "loss": 3.47394437789917, "lr": 0.0006, "tok_s": 53735.04933796688} +{"step": 1040, "loss": 3.447183310985565, "lr": 0.0006, "tok_s": 54384.39453636309} +{"step": 1060, "loss": 3.458915412425995, "lr": 0.0006, "tok_s": 54382.80339353018} +{"step": 1080, "loss": 3.4484529376029966, "lr": 0.0006, "tok_s": 54378.62864135084} +{"step": 1100, "loss": 3.433352470397949, "lr": 0.0006, "tok_s": 54381.845766923994} +{"step": 1120, "loss": 3.4153954148292542, "lr": 0.0006, "tok_s": 54378.996554840145} +{"step": 1140, "loss": 3.4083414316177367, "lr": 0.0006, "tok_s": 54382.7574648779} +{"step": 1160, "loss": 3.403551959991455, "lr": 0.0006, "tok_s": 54383.10808469444} +{"step": 1180, "loss": 3.40577654838562, "lr": 0.0006, "tok_s": 54380.83626420643} +{"step": 1200, "loss": 3.3882885336875916, "lr": 0.0006, "tok_s": 54381.06421098787} +{"step": 1220, "loss": 3.385551917552948, "lr": 0.0006, "tok_s": 54376.27725923776} +{"step": 1240, "loss": 3.3774680852890016, "lr": 0.0006, "tok_s": 54378.30409899} +{"step": 1260, "loss": 3.364372217655182, "lr": 0.0006, "tok_s": 54380.226129333794} +{"step": 1280, "loss": 3.3675967693328857, "lr": 0.0006, "tok_s": 54382.87803616131} +{"step": 1300, "loss": 3.3608749508857727, "lr": 0.0006, "tok_s": 54380.05951123605} +{"step": 1320, "loss": 3.3530895113945007, "lr": 0.0006, "tok_s": 54380.28624119837} +{"step": 1340, "loss": 3.3537341475486757, "lr": 0.0006, "tok_s": 54379.937943873556} +{"step": 1360, "loss": 3.3449841737747192, "lr": 0.0006, "tok_s": 54378.28446655379} +{"step": 1380, "loss": 3.3367194056510927, "lr": 0.0006, "tok_s": 54380.54074312651} +{"step": 1400, "loss": 3.3268325209617613, "lr": 0.0006, "tok_s": 54381.70583428245} +{"step": 1420, "loss": 3.3142680764198302, "lr": 0.0006, "tok_s": 54385.200869299784} +{"step": 1440, "loss": 3.321825695037842, "lr": 0.0006, "tok_s": 54383.38130768183} +{"step": 1460, "loss": 3.3194995641708376, "lr": 0.0006, "tok_s": 54381.14503505483} +{"step": 1480, "loss": 3.323328506946564, "lr": 0.0006, "tok_s": 54379.48812246335} +{"step": 1500, "loss": 3.2963005781173704, "lr": 0.0006, "tok_s": 54380.93403229694} +{"step": 1520, "loss": 3.304063355922699, "lr": 0.0006, "tok_s": 53713.09851579601} +{"step": 1540, "loss": 3.28833954334259, "lr": 0.0006, "tok_s": 54373.54082492265} +{"step": 1560, "loss": 3.2969040393829347, "lr": 0.0006, "tok_s": 54376.58463371382} +{"step": 1580, "loss": 3.2850605249404907, "lr": 0.0006, "tok_s": 54375.742454756866} +{"step": 1600, "loss": 3.2749640941619873, "lr": 0.0006, "tok_s": 54377.58006143249} +{"step": 1620, "loss": 3.2752556204795837, "lr": 0.0006, "tok_s": 54376.30650403504} +{"step": 1640, "loss": 3.2726037979125975, "lr": 0.0006, "tok_s": 40751.557960654085} +{"step": 1660, "loss": 3.273281013965607, "lr": 0.0006, "tok_s": 39289.77490753473} +{"step": 1680, "loss": 3.26226863861084, "lr": 0.0006, "tok_s": 40313.389668948854} +{"step": 1700, "loss": 3.2555036783218383, "lr": 0.0006, "tok_s": 54379.78369934085} +{"step": 1720, "loss": 3.25777188539505, "lr": 0.0006, "tok_s": 54379.535726529546} +{"step": 1740, "loss": 3.251254212856293, "lr": 0.0006, "tok_s": 54384.92964555243} +{"step": 1760, "loss": 3.2507732033729555, "lr": 0.0006, "tok_s": 54388.55746184499} +{"step": 1780, "loss": 3.238064002990723, "lr": 0.0006, "tok_s": 54383.73502894239} +{"step": 1800, "loss": 3.2417608499526978, "lr": 0.0006, "tok_s": 54379.47595251065} +{"step": 1820, "loss": 3.2374199867248534, "lr": 0.0006, "tok_s": 53645.94184194194} +{"step": 1840, "loss": 3.2312928795814515, "lr": 0.0006, "tok_s": 30466.846568588197} +{"step": 1860, "loss": 3.230325162410736, "lr": 0.0006, "tok_s": 44403.16651628122} +{"step": 1880, "loss": 3.2265440702438353, "lr": 0.0006, "tok_s": 31623.168463345162} +{"step": 1900, "loss": 3.2208963513374327, "lr": 0.0006, "tok_s": 31365.17852899342} +{"step": 1920, "loss": 3.2205076932907106, "lr": 0.0006, "tok_s": 29664.547690026135} +{"step": 1940, "loss": 3.2140785694122314, "lr": 0.0006, "tok_s": 29662.386463764484} +{"step": 1960, "loss": 3.2180041909217834, "lr": 0.0006, "tok_s": 30456.666177317657} +{"step": 1980, "loss": 3.2126453638076784, "lr": 0.0006, "tok_s": 32075.633695387703} +{"step": 2000, "loss": 3.2058145880699156, "lr": 0.0006, "tok_s": 32046.597929714466} +{"step": 2020, "loss": 3.2054898142814636, "lr": 0.0006, "tok_s": 31786.004615394697} +{"step": 2040, "loss": 3.1983474612236025, "lr": 0.0006, "tok_s": 31965.00091165428} +{"step": 2060, "loss": 3.1985918998718263, "lr": 0.0006, "tok_s": 31994.541837655575} +{"step": 2080, "loss": 3.187583899497986, "lr": 0.0006, "tok_s": 32010.290784090703} +{"step": 2100, "loss": 3.196134352684021, "lr": 0.0006, "tok_s": 31946.15269744886} +{"step": 2120, "loss": 3.1858644843101502, "lr": 0.0006, "tok_s": 31941.206322038823} +{"step": 2140, "loss": 3.1989166855812075, "lr": 0.0006, "tok_s": 32012.582473647} +{"step": 2160, "loss": 3.1776081800460814, "lr": 0.0006, "tok_s": 31944.20210738539} +{"step": 2180, "loss": 3.179273557662964, "lr": 0.0006, "tok_s": 31909.084135585163} +{"step": 2200, "loss": 3.180311906337738, "lr": 0.0006, "tok_s": 39315.18392988774} +{"step": 2220, "loss": 3.1733631134033202, "lr": 0.0006, "tok_s": 34362.62429152448} +{"step": 2240, "loss": 3.170001435279846, "lr": 0.0006, "tok_s": 28500.10494578955} +{"step": 2260, "loss": 3.166220474243164, "lr": 0.0006, "tok_s": 32518.169542876723} +{"step": 2280, "loss": 3.1624667048454285, "lr": 0.0006, "tok_s": 30345.85548236686} +{"step": 2300, "loss": 3.165924310684204, "lr": 0.0006, "tok_s": 29311.251041075742} +{"step": 2320, "loss": 3.1626460313797, "lr": 0.0006, "tok_s": 34479.854490218255} +{"step": 2340, "loss": 3.157814919948578, "lr": 0.0006, "tok_s": 28009.653677274284} diff --git a/MU2/run/srodowisko_MU2_mini0_resume0.txt b/MU2/run/srodowisko_MU2_mini0_resume0.txt new file mode 100644 index 0000000000000000000000000000000000000000..ff2a10e3cb5a8422e349b2cdc8f69eb58e064852 --- /dev/null +++ b/MU2/run/srodowisko_MU2_mini0_resume0.txt @@ -0,0 +1,64 @@ +data 2026-10-23T02:48:25Z ARM MU2 GPU 0 MINI 0 DATA_SEED 20261022 CKPT 500 RESUME 0 DEST /var/lib/docker/kolektyw/dowody400m_ckpt/wyniki_MU2 +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +0702c32c0ad602d8d12b16539e05cfbc68d1823cfd39b82b984357d78f27f664 /home/ubuntu/kolektyw/dowody400m/kod/ramie_v03.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +341aae81ce8512294dcd7de93ced26053025677b4ab449606be989f7fd898d27 /home/ubuntu/kolektyw/dowody400m/kod/kopiarka_ckpt.sh +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 15082 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 14222 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 15482 MiB, 595.104.02 diff --git a/MU2/run/srodowisko_resume0.txt b/MU2/run/srodowisko_resume0.txt new file mode 100644 index 0000000000000000000000000000000000000000..ff2a10e3cb5a8422e349b2cdc8f69eb58e064852 --- /dev/null +++ b/MU2/run/srodowisko_resume0.txt @@ -0,0 +1,64 @@ +data 2026-10-23T02:48:25Z ARM MU2 GPU 0 MINI 0 DATA_SEED 20261022 CKPT 500 RESUME 0 DEST /var/lib/docker/kolektyw/dowody400m_ckpt/wyniki_MU2 +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +0702c32c0ad602d8d12b16539e05cfbc68d1823cfd39b82b984357d78f27f664 /home/ubuntu/kolektyw/dowody400m/kod/ramie_v03.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +341aae81ce8512294dcd7de93ced26053025677b4ab449606be989f7fd898d27 /home/ubuntu/kolektyw/dowody400m/kod/kopiarka_ckpt.sh +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 15082 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 14222 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 15482 MiB, 595.104.02 diff --git a/MU2/run/train.log b/MU2/run/train.log new file mode 100644 index 0000000000000000000000000000000000000000..6f5607637bea8013e53f79a708752740d802a1cc --- /dev/null +++ b/MU2/run/train.log @@ -0,0 +1,129 @@ +[04:48:32] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[04:48:32] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[04:48:34] model GPT-ref: 183.6M param (L16 d896 h14) +[04:48:34] optimizer=muon muon_lr=0.04 (mult=66.7x) +[04:48:34] r11 pamiec: act_ckpt=True loss_chunk=1024 +[04:48:34] r13: mixer_layout=AAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[04:48:35] torch.compile enabled +[04:48:35] START gpt-ref: steps=4768 (od 0) lr=0.0006->6e-05 +[04:52:32] step 20/4768 loss 10.2553 lr 2.40e-05 gnorm 2.32 44,304 tok/s peakVRAM 13.7GB +[04:55:44] step 40/4768 loss 9.2968 lr 4.80e-05 gnorm 1.00 54,450 tok/s peakVRAM 13.7GB +[04:58:57] step 60/4768 loss 8.4951 lr 7.20e-05 gnorm 1.46 54,449 tok/s peakVRAM 13.7GB +[05:02:10] step 80/4768 loss 7.8249 lr 9.60e-05 gnorm 0.69 54,439 tok/s peakVRAM 13.7GB +[05:05:22] step 100/4768 loss 7.3027 lr 1.20e-04 gnorm 0.65 54,421 tok/s peakVRAM 13.7GB +[05:08:35] step 120/4768 loss 6.7967 lr 1.44e-04 gnorm 0.71 54,420 tok/s peakVRAM 13.7GB +[05:11:48] step 140/4768 loss 6.4050 lr 1.68e-04 gnorm 0.49 54,415 tok/s peakVRAM 13.7GB +[05:15:00] step 160/4768 loss 6.1028 lr 1.92e-04 gnorm 0.54 54,413 tok/s peakVRAM 13.7GB +[05:18:13] step 180/4768 loss 5.8786 lr 2.16e-04 gnorm 0.51 54,414 tok/s peakVRAM 13.7GB +[05:21:26] step 200/4768 loss 5.6752 lr 2.40e-04 gnorm 0.56 54,412 tok/s peakVRAM 13.7GB +[05:24:38] step 220/4768 loss 5.4946 lr 2.64e-04 gnorm 0.62 54,411 tok/s peakVRAM 13.7GB +[05:27:51] step 240/4768 loss 5.3280 lr 2.88e-04 gnorm 0.57 54,408 tok/s peakVRAM 13.7GB +[05:31:04] step 260/4768 loss 5.1680 lr 3.12e-04 gnorm 0.43 54,408 tok/s peakVRAM 13.7GB +[05:34:17] step 280/4768 loss 5.0263 lr 3.36e-04 gnorm 0.37 54,403 tok/s peakVRAM 13.7GB +[05:37:29] step 300/4768 loss 4.8846 lr 3.60e-04 gnorm 0.45 54,404 tok/s peakVRAM 13.7GB +[05:40:42] step 320/4768 loss 4.7472 lr 3.84e-04 gnorm 0.32 54,402 tok/s peakVRAM 13.7GB +[05:43:55] step 340/4768 loss 4.6110 lr 4.08e-04 gnorm 0.33 54,399 tok/s peakVRAM 13.7GB +[05:47:08] step 360/4768 loss 4.5043 lr 4.32e-04 gnorm 0.26 54,397 tok/s peakVRAM 13.7GB +[05:50:20] step 380/4768 loss 4.4053 lr 4.56e-04 gnorm 0.22 54,390 tok/s peakVRAM 13.7GB +[05:53:33] step 400/4768 loss 4.3194 lr 4.80e-04 gnorm 0.25 54,389 tok/s peakVRAM 13.7GB +[05:56:46] step 420/4768 loss 4.2544 lr 5.04e-04 gnorm 0.21 54,392 tok/s peakVRAM 13.7GB +[05:59:59] step 440/4768 loss 4.1844 lr 5.28e-04 gnorm 0.21 54,392 tok/s peakVRAM 13.7GB +[06:03:12] step 460/4768 loss 4.1207 lr 5.52e-04 gnorm 0.18 54,390 tok/s peakVRAM 13.7GB +[06:06:24] step 480/4768 loss 4.0799 lr 5.76e-04 gnorm 0.22 54,389 tok/s peakVRAM 13.7GB +[06:09:37] step 500/4768 loss 4.0291 lr 6.00e-04 gnorm 0.18 54,386 tok/s peakVRAM 13.7GB +[06:09:39] ckpt @ 500 +[06:12:52] step 520/4768 loss 3.9946 lr 6.00e-04 gnorm 0.16 53,918 tok/s peakVRAM 13.7GB +[06:16:04] step 540/4768 loss 3.9497 lr 6.00e-04 gnorm 0.16 54,388 tok/s peakVRAM 13.7GB +[06:19:17] step 560/4768 loss 3.8994 lr 6.00e-04 gnorm 0.20 54,386 tok/s peakVRAM 13.7GB +[06:22:30] step 580/4768 loss 3.8611 lr 6.00e-04 gnorm 0.15 54,384 tok/s peakVRAM 13.7GB +[06:25:43] step 600/4768 loss 3.8206 lr 6.00e-04 gnorm 0.15 54,386 tok/s peakVRAM 13.7GB +[06:28:56] step 620/4768 loss 3.7999 lr 6.00e-04 gnorm 0.14 54,384 tok/s peakVRAM 13.7GB +[06:32:09] step 640/4768 loss 3.7752 lr 6.00e-04 gnorm 0.12 54,382 tok/s peakVRAM 13.7GB +[06:35:21] step 660/4768 loss 3.7472 lr 6.00e-04 gnorm 0.11 54,379 tok/s peakVRAM 13.7GB +[06:38:34] step 680/4768 loss 3.7249 lr 6.00e-04 gnorm 0.13 54,376 tok/s peakVRAM 13.7GB +[06:41:47] step 700/4768 loss 3.6959 lr 6.00e-04 gnorm 0.11 54,382 tok/s peakVRAM 13.7GB +[06:45:00] step 720/4768 loss 3.6670 lr 6.00e-04 gnorm 0.10 54,381 tok/s peakVRAM 13.7GB +[06:48:13] step 740/4768 loss 3.6638 lr 6.00e-04 gnorm 0.12 54,382 tok/s peakVRAM 13.7GB +[06:51:25] step 760/4768 loss 3.6386 lr 6.00e-04 gnorm 0.12 54,383 tok/s peakVRAM 13.7GB +[06:54:38] step 780/4768 loss 3.6130 lr 6.00e-04 gnorm 0.10 54,386 tok/s peakVRAM 13.7GB +[06:57:51] step 800/4768 loss 3.6005 lr 6.00e-04 gnorm 0.12 54,385 tok/s peakVRAM 13.7GB +[07:01:04] step 820/4768 loss 3.5843 lr 6.00e-04 gnorm 0.10 54,385 tok/s peakVRAM 13.7GB +[07:04:17] step 840/4768 loss 3.5713 lr 6.00e-04 gnorm 0.11 54,382 tok/s peakVRAM 13.7GB +[07:07:29] step 860/4768 loss 3.5506 lr 6.00e-04 gnorm 0.10 54,384 tok/s peakVRAM 13.7GB +[07:10:42] step 880/4768 loss 3.5451 lr 6.00e-04 gnorm 0.11 54,385 tok/s peakVRAM 13.7GB +[07:13:55] step 900/4768 loss 3.5280 lr 6.00e-04 gnorm 0.11 54,387 tok/s peakVRAM 13.7GB +[07:17:08] step 920/4768 loss 3.5207 lr 6.00e-04 gnorm 0.10 54,382 tok/s peakVRAM 13.7GB +[07:20:21] step 940/4768 loss 3.5164 lr 6.00e-04 gnorm 0.10 54,383 tok/s peakVRAM 13.7GB +[07:23:34] step 960/4768 loss 3.4913 lr 6.00e-04 gnorm 0.08 54,381 tok/s peakVRAM 13.7GB +[07:26:46] step 980/4768 loss 3.4890 lr 6.00e-04 gnorm 0.08 54,380 tok/s peakVRAM 13.7GB +[07:29:59] step 1000/4768 loss 3.4788 lr 6.00e-04 gnorm 0.09 54,379 tok/s peakVRAM 13.7GB +[07:30:02] ckpt @ 1000 +[07:33:14] step 1020/4768 loss 3.4739 lr 6.00e-04 gnorm 0.12 53,735 tok/s peakVRAM 13.7GB +[07:36:27] step 1040/4768 loss 3.4472 lr 6.00e-04 gnorm 0.09 54,384 tok/s peakVRAM 13.7GB +[07:39:40] step 1060/4768 loss 3.4589 lr 6.00e-04 gnorm 0.10 54,383 tok/s peakVRAM 13.7GB +[07:42:53] step 1080/4768 loss 3.4485 lr 6.00e-04 gnorm 0.09 54,379 tok/s peakVRAM 13.7GB +[07:46:06] step 1100/4768 loss 3.4334 lr 6.00e-04 gnorm 0.11 54,382 tok/s peakVRAM 13.7GB +[07:49:18] step 1120/4768 loss 3.4154 lr 6.00e-04 gnorm 0.10 54,379 tok/s peakVRAM 13.7GB +[07:52:31] step 1140/4768 loss 3.4083 lr 6.00e-04 gnorm 0.10 54,383 tok/s peakVRAM 13.7GB +[07:55:44] step 1160/4768 loss 3.4036 lr 6.00e-04 gnorm 0.09 54,383 tok/s peakVRAM 13.7GB +[07:58:57] step 1180/4768 loss 3.4058 lr 6.00e-04 gnorm 0.09 54,381 tok/s peakVRAM 13.7GB +[08:02:10] step 1200/4768 loss 3.3883 lr 6.00e-04 gnorm 0.09 54,381 tok/s peakVRAM 13.7GB +[08:05:23] step 1220/4768 loss 3.3856 lr 6.00e-04 gnorm 0.10 54,376 tok/s peakVRAM 13.7GB +[08:08:35] step 1240/4768 loss 3.3775 lr 6.00e-04 gnorm 0.09 54,378 tok/s peakVRAM 13.7GB +[08:11:48] step 1260/4768 loss 3.3644 lr 6.00e-04 gnorm 0.10 54,380 tok/s peakVRAM 13.7GB +[08:15:01] step 1280/4768 loss 3.3676 lr 6.00e-04 gnorm 0.10 54,383 tok/s peakVRAM 13.7GB +[08:18:14] step 1300/4768 loss 3.3609 lr 6.00e-04 gnorm 0.09 54,380 tok/s peakVRAM 13.7GB +[08:21:27] step 1320/4768 loss 3.3531 lr 6.00e-04 gnorm 0.10 54,380 tok/s peakVRAM 13.7GB +[08:24:39] step 1340/4768 loss 3.3537 lr 6.00e-04 gnorm 0.09 54,380 tok/s peakVRAM 13.7GB +[08:27:52] step 1360/4768 loss 3.3450 lr 6.00e-04 gnorm 0.09 54,378 tok/s peakVRAM 13.7GB +[08:31:05] step 1380/4768 loss 3.3367 lr 6.00e-04 gnorm 0.10 54,381 tok/s peakVRAM 13.7GB +[08:34:18] step 1400/4768 loss 3.3268 lr 6.00e-04 gnorm 0.09 54,382 tok/s peakVRAM 13.7GB +[08:37:31] step 1420/4768 loss 3.3143 lr 6.00e-04 gnorm 0.10 54,385 tok/s peakVRAM 13.7GB +[08:40:44] step 1440/4768 loss 3.3218 lr 6.00e-04 gnorm 0.08 54,383 tok/s peakVRAM 13.7GB +[08:43:56] step 1460/4768 loss 3.3195 lr 6.00e-04 gnorm 0.09 54,381 tok/s peakVRAM 13.7GB +[08:47:09] step 1480/4768 loss 3.3233 lr 6.00e-04 gnorm 0.08 54,379 tok/s peakVRAM 13.7GB +[08:50:22] step 1500/4768 loss 3.2963 lr 6.00e-04 gnorm 0.10 54,381 tok/s peakVRAM 13.7GB +[08:50:24] ckpt @ 1500 +[08:53:37] step 1520/4768 loss 3.3041 lr 6.00e-04 gnorm 0.09 53,713 tok/s peakVRAM 13.7GB +[08:56:50] step 1540/4768 loss 3.2883 lr 6.00e-04 gnorm 0.09 54,374 tok/s peakVRAM 13.7GB +[09:00:03] step 1560/4768 loss 3.2969 lr 6.00e-04 gnorm 0.08 54,377 tok/s peakVRAM 13.7GB +[09:03:16] step 1580/4768 loss 3.2851 lr 6.00e-04 gnorm 0.09 54,376 tok/s peakVRAM 13.7GB +[09:06:29] step 1600/4768 loss 3.2750 lr 6.00e-04 gnorm 0.09 54,378 tok/s peakVRAM 13.7GB +[09:09:41] step 1620/4768 loss 3.2753 lr 6.00e-04 gnorm 0.09 54,376 tok/s peakVRAM 13.7GB +[09:13:59] step 1640/4768 loss 3.2726 lr 6.00e-04 gnorm 0.09 40,752 tok/s peakVRAM 13.7GB +[09:18:26] step 1660/4768 loss 3.2733 lr 6.00e-04 gnorm 0.09 39,290 tok/s peakVRAM 13.7GB +[09:22:46] step 1680/4768 loss 3.2623 lr 6.00e-04 gnorm 0.10 40,313 tok/s peakVRAM 13.7GB +[09:25:59] step 1700/4768 loss 3.2555 lr 6.00e-04 gnorm 0.09 54,380 tok/s peakVRAM 13.7GB +[09:29:11] step 1720/4768 loss 3.2578 lr 6.00e-04 gnorm 0.09 54,380 tok/s peakVRAM 13.7GB +[09:32:24] step 1740/4768 loss 3.2513 lr 6.00e-04 gnorm 0.10 54,385 tok/s peakVRAM 13.7GB +[09:35:37] step 1760/4768 loss 3.2508 lr 6.00e-04 gnorm 0.10 54,389 tok/s peakVRAM 13.7GB +[09:38:50] step 1780/4768 loss 3.2381 lr 6.00e-04 gnorm 0.10 54,384 tok/s peakVRAM 13.7GB +[09:42:03] step 1800/4768 loss 3.2418 lr 6.00e-04 gnorm 0.10 54,379 tok/s peakVRAM 13.7GB +[09:45:18] step 1820/4768 loss 3.2374 lr 6.00e-04 gnorm 0.08 53,646 tok/s peakVRAM 13.7GB +[09:51:02] step 1840/4768 loss 3.2313 lr 6.00e-04 gnorm 0.09 30,467 tok/s peakVRAM 13.7GB +[09:54:58] step 1860/4768 loss 3.2303 lr 6.00e-04 gnorm 0.09 44,403 tok/s peakVRAM 13.7GB +[10:00:30] step 1880/4768 loss 3.2265 lr 6.00e-04 gnorm 0.08 31,623 tok/s peakVRAM 13.7GB +[10:06:04] step 1900/4768 loss 3.2209 lr 6.00e-04 gnorm 0.09 31,365 tok/s peakVRAM 13.7GB +[10:11:58] step 1920/4768 loss 3.2205 lr 6.00e-04 gnorm 0.08 29,665 tok/s peakVRAM 13.7GB +[10:17:51] step 1940/4768 loss 3.2141 lr 6.00e-04 gnorm 0.08 29,662 tok/s peakVRAM 13.7GB +[10:23:36] step 1960/4768 loss 3.2180 lr 6.00e-04 gnorm 0.08 30,457 tok/s peakVRAM 13.7GB +[10:29:02] step 1980/4768 loss 3.2126 lr 6.00e-04 gnorm 0.09 32,076 tok/s peakVRAM 13.7GB +[10:34:30] step 2000/4768 loss 3.2058 lr 6.00e-04 gnorm 0.09 32,047 tok/s peakVRAM 13.7GB +[10:34:32] ckpt @ 2000 +[10:40:00] step 2020/4768 loss 3.2055 lr 6.00e-04 gnorm 0.09 31,786 tok/s peakVRAM 13.7GB +[10:45:28] step 2040/4768 loss 3.1983 lr 6.00e-04 gnorm 0.09 31,965 tok/s peakVRAM 13.7GB +[10:50:55] step 2060/4768 loss 3.1986 lr 6.00e-04 gnorm 0.08 31,995 tok/s peakVRAM 13.7GB +[10:56:23] step 2080/4768 loss 3.1876 lr 6.00e-04 gnorm 0.08 32,010 tok/s peakVRAM 13.7GB +[11:01:51] step 2100/4768 loss 3.1961 lr 6.00e-04 gnorm 0.10 31,946 tok/s peakVRAM 13.7GB +[11:07:19] step 2120/4768 loss 3.1859 lr 6.00e-04 gnorm 0.09 31,941 tok/s peakVRAM 13.7GB +[11:12:47] step 2140/4768 loss 3.1989 lr 6.00e-04 gnorm 0.08 32,013 tok/s peakVRAM 13.7GB +[11:18:15] step 2160/4768 loss 3.1776 lr 6.00e-04 gnorm 0.08 31,944 tok/s peakVRAM 13.7GB +[11:23:44] step 2180/4768 loss 3.1793 lr 6.00e-04 gnorm 0.09 31,909 tok/s peakVRAM 13.7GB +[11:28:11] step 2200/4768 loss 3.1803 lr 6.00e-04 gnorm 0.08 39,315 tok/s peakVRAM 13.7GB +[11:33:16] step 2220/4768 loss 3.1734 lr 6.00e-04 gnorm 0.09 34,363 tok/s peakVRAM 13.7GB +[11:39:24] step 2240/4768 loss 3.1700 lr 6.00e-04 gnorm 0.08 28,500 tok/s peakVRAM 13.7GB +[11:44:46] step 2260/4768 loss 3.1662 lr 6.00e-04 gnorm 0.08 32,518 tok/s peakVRAM 13.7GB +[11:50:32] step 2280/4768 loss 3.1625 lr 6.00e-04 gnorm 0.08 30,346 tok/s peakVRAM 13.7GB +[11:56:29] step 2300/4768 loss 3.1659 lr 6.00e-04 gnorm 0.08 29,311 tok/s peakVRAM 13.7GB +[12:01:34] step 2320/4768 loss 3.1626 lr 6.00e-04 gnorm 0.08 34,480 tok/s peakVRAM 13.7GB +[12:07:48] step 2340/4768 loss 3.1578 lr 6.00e-04 gnorm 0.09 28,010 tok/s peakVRAM 13.7GB diff --git a/MU2/run/train.out b/MU2/run/train.out new file mode 100644 index 0000000000000000000000000000000000000000..6f5607637bea8013e53f79a708752740d802a1cc --- /dev/null +++ b/MU2/run/train.out @@ -0,0 +1,129 @@ +[04:48:32] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[04:48:32] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[04:48:34] model GPT-ref: 183.6M param (L16 d896 h14) +[04:48:34] optimizer=muon muon_lr=0.04 (mult=66.7x) +[04:48:34] r11 pamiec: act_ckpt=True loss_chunk=1024 +[04:48:34] r13: mixer_layout=AAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[04:48:35] torch.compile enabled +[04:48:35] START gpt-ref: steps=4768 (od 0) lr=0.0006->6e-05 +[04:52:32] step 20/4768 loss 10.2553 lr 2.40e-05 gnorm 2.32 44,304 tok/s peakVRAM 13.7GB +[04:55:44] step 40/4768 loss 9.2968 lr 4.80e-05 gnorm 1.00 54,450 tok/s peakVRAM 13.7GB +[04:58:57] step 60/4768 loss 8.4951 lr 7.20e-05 gnorm 1.46 54,449 tok/s peakVRAM 13.7GB +[05:02:10] step 80/4768 loss 7.8249 lr 9.60e-05 gnorm 0.69 54,439 tok/s peakVRAM 13.7GB +[05:05:22] step 100/4768 loss 7.3027 lr 1.20e-04 gnorm 0.65 54,421 tok/s peakVRAM 13.7GB +[05:08:35] step 120/4768 loss 6.7967 lr 1.44e-04 gnorm 0.71 54,420 tok/s peakVRAM 13.7GB +[05:11:48] step 140/4768 loss 6.4050 lr 1.68e-04 gnorm 0.49 54,415 tok/s peakVRAM 13.7GB +[05:15:00] step 160/4768 loss 6.1028 lr 1.92e-04 gnorm 0.54 54,413 tok/s peakVRAM 13.7GB +[05:18:13] step 180/4768 loss 5.8786 lr 2.16e-04 gnorm 0.51 54,414 tok/s peakVRAM 13.7GB +[05:21:26] step 200/4768 loss 5.6752 lr 2.40e-04 gnorm 0.56 54,412 tok/s peakVRAM 13.7GB +[05:24:38] step 220/4768 loss 5.4946 lr 2.64e-04 gnorm 0.62 54,411 tok/s peakVRAM 13.7GB +[05:27:51] step 240/4768 loss 5.3280 lr 2.88e-04 gnorm 0.57 54,408 tok/s peakVRAM 13.7GB +[05:31:04] step 260/4768 loss 5.1680 lr 3.12e-04 gnorm 0.43 54,408 tok/s peakVRAM 13.7GB +[05:34:17] step 280/4768 loss 5.0263 lr 3.36e-04 gnorm 0.37 54,403 tok/s peakVRAM 13.7GB +[05:37:29] step 300/4768 loss 4.8846 lr 3.60e-04 gnorm 0.45 54,404 tok/s peakVRAM 13.7GB +[05:40:42] step 320/4768 loss 4.7472 lr 3.84e-04 gnorm 0.32 54,402 tok/s peakVRAM 13.7GB +[05:43:55] step 340/4768 loss 4.6110 lr 4.08e-04 gnorm 0.33 54,399 tok/s peakVRAM 13.7GB +[05:47:08] step 360/4768 loss 4.5043 lr 4.32e-04 gnorm 0.26 54,397 tok/s peakVRAM 13.7GB +[05:50:20] step 380/4768 loss 4.4053 lr 4.56e-04 gnorm 0.22 54,390 tok/s peakVRAM 13.7GB +[05:53:33] step 400/4768 loss 4.3194 lr 4.80e-04 gnorm 0.25 54,389 tok/s peakVRAM 13.7GB +[05:56:46] step 420/4768 loss 4.2544 lr 5.04e-04 gnorm 0.21 54,392 tok/s peakVRAM 13.7GB +[05:59:59] step 440/4768 loss 4.1844 lr 5.28e-04 gnorm 0.21 54,392 tok/s peakVRAM 13.7GB +[06:03:12] step 460/4768 loss 4.1207 lr 5.52e-04 gnorm 0.18 54,390 tok/s peakVRAM 13.7GB +[06:06:24] step 480/4768 loss 4.0799 lr 5.76e-04 gnorm 0.22 54,389 tok/s peakVRAM 13.7GB +[06:09:37] step 500/4768 loss 4.0291 lr 6.00e-04 gnorm 0.18 54,386 tok/s peakVRAM 13.7GB +[06:09:39] ckpt @ 500 +[06:12:52] step 520/4768 loss 3.9946 lr 6.00e-04 gnorm 0.16 53,918 tok/s peakVRAM 13.7GB +[06:16:04] step 540/4768 loss 3.9497 lr 6.00e-04 gnorm 0.16 54,388 tok/s peakVRAM 13.7GB +[06:19:17] step 560/4768 loss 3.8994 lr 6.00e-04 gnorm 0.20 54,386 tok/s peakVRAM 13.7GB +[06:22:30] step 580/4768 loss 3.8611 lr 6.00e-04 gnorm 0.15 54,384 tok/s peakVRAM 13.7GB +[06:25:43] step 600/4768 loss 3.8206 lr 6.00e-04 gnorm 0.15 54,386 tok/s peakVRAM 13.7GB +[06:28:56] step 620/4768 loss 3.7999 lr 6.00e-04 gnorm 0.14 54,384 tok/s peakVRAM 13.7GB +[06:32:09] step 640/4768 loss 3.7752 lr 6.00e-04 gnorm 0.12 54,382 tok/s peakVRAM 13.7GB +[06:35:21] step 660/4768 loss 3.7472 lr 6.00e-04 gnorm 0.11 54,379 tok/s peakVRAM 13.7GB +[06:38:34] step 680/4768 loss 3.7249 lr 6.00e-04 gnorm 0.13 54,376 tok/s peakVRAM 13.7GB +[06:41:47] step 700/4768 loss 3.6959 lr 6.00e-04 gnorm 0.11 54,382 tok/s peakVRAM 13.7GB +[06:45:00] step 720/4768 loss 3.6670 lr 6.00e-04 gnorm 0.10 54,381 tok/s peakVRAM 13.7GB +[06:48:13] step 740/4768 loss 3.6638 lr 6.00e-04 gnorm 0.12 54,382 tok/s peakVRAM 13.7GB +[06:51:25] step 760/4768 loss 3.6386 lr 6.00e-04 gnorm 0.12 54,383 tok/s peakVRAM 13.7GB +[06:54:38] step 780/4768 loss 3.6130 lr 6.00e-04 gnorm 0.10 54,386 tok/s peakVRAM 13.7GB +[06:57:51] step 800/4768 loss 3.6005 lr 6.00e-04 gnorm 0.12 54,385 tok/s peakVRAM 13.7GB +[07:01:04] step 820/4768 loss 3.5843 lr 6.00e-04 gnorm 0.10 54,385 tok/s peakVRAM 13.7GB +[07:04:17] step 840/4768 loss 3.5713 lr 6.00e-04 gnorm 0.11 54,382 tok/s peakVRAM 13.7GB +[07:07:29] step 860/4768 loss 3.5506 lr 6.00e-04 gnorm 0.10 54,384 tok/s peakVRAM 13.7GB +[07:10:42] step 880/4768 loss 3.5451 lr 6.00e-04 gnorm 0.11 54,385 tok/s peakVRAM 13.7GB +[07:13:55] step 900/4768 loss 3.5280 lr 6.00e-04 gnorm 0.11 54,387 tok/s peakVRAM 13.7GB +[07:17:08] step 920/4768 loss 3.5207 lr 6.00e-04 gnorm 0.10 54,382 tok/s peakVRAM 13.7GB +[07:20:21] step 940/4768 loss 3.5164 lr 6.00e-04 gnorm 0.10 54,383 tok/s peakVRAM 13.7GB +[07:23:34] step 960/4768 loss 3.4913 lr 6.00e-04 gnorm 0.08 54,381 tok/s peakVRAM 13.7GB +[07:26:46] step 980/4768 loss 3.4890 lr 6.00e-04 gnorm 0.08 54,380 tok/s peakVRAM 13.7GB +[07:29:59] step 1000/4768 loss 3.4788 lr 6.00e-04 gnorm 0.09 54,379 tok/s peakVRAM 13.7GB +[07:30:02] ckpt @ 1000 +[07:33:14] step 1020/4768 loss 3.4739 lr 6.00e-04 gnorm 0.12 53,735 tok/s peakVRAM 13.7GB +[07:36:27] step 1040/4768 loss 3.4472 lr 6.00e-04 gnorm 0.09 54,384 tok/s peakVRAM 13.7GB +[07:39:40] step 1060/4768 loss 3.4589 lr 6.00e-04 gnorm 0.10 54,383 tok/s peakVRAM 13.7GB +[07:42:53] step 1080/4768 loss 3.4485 lr 6.00e-04 gnorm 0.09 54,379 tok/s peakVRAM 13.7GB +[07:46:06] step 1100/4768 loss 3.4334 lr 6.00e-04 gnorm 0.11 54,382 tok/s peakVRAM 13.7GB +[07:49:18] step 1120/4768 loss 3.4154 lr 6.00e-04 gnorm 0.10 54,379 tok/s peakVRAM 13.7GB +[07:52:31] step 1140/4768 loss 3.4083 lr 6.00e-04 gnorm 0.10 54,383 tok/s peakVRAM 13.7GB +[07:55:44] step 1160/4768 loss 3.4036 lr 6.00e-04 gnorm 0.09 54,383 tok/s peakVRAM 13.7GB +[07:58:57] step 1180/4768 loss 3.4058 lr 6.00e-04 gnorm 0.09 54,381 tok/s peakVRAM 13.7GB +[08:02:10] step 1200/4768 loss 3.3883 lr 6.00e-04 gnorm 0.09 54,381 tok/s peakVRAM 13.7GB +[08:05:23] step 1220/4768 loss 3.3856 lr 6.00e-04 gnorm 0.10 54,376 tok/s peakVRAM 13.7GB +[08:08:35] step 1240/4768 loss 3.3775 lr 6.00e-04 gnorm 0.09 54,378 tok/s peakVRAM 13.7GB +[08:11:48] step 1260/4768 loss 3.3644 lr 6.00e-04 gnorm 0.10 54,380 tok/s peakVRAM 13.7GB +[08:15:01] step 1280/4768 loss 3.3676 lr 6.00e-04 gnorm 0.10 54,383 tok/s peakVRAM 13.7GB +[08:18:14] step 1300/4768 loss 3.3609 lr 6.00e-04 gnorm 0.09 54,380 tok/s peakVRAM 13.7GB +[08:21:27] step 1320/4768 loss 3.3531 lr 6.00e-04 gnorm 0.10 54,380 tok/s peakVRAM 13.7GB +[08:24:39] step 1340/4768 loss 3.3537 lr 6.00e-04 gnorm 0.09 54,380 tok/s peakVRAM 13.7GB +[08:27:52] step 1360/4768 loss 3.3450 lr 6.00e-04 gnorm 0.09 54,378 tok/s peakVRAM 13.7GB +[08:31:05] step 1380/4768 loss 3.3367 lr 6.00e-04 gnorm 0.10 54,381 tok/s peakVRAM 13.7GB +[08:34:18] step 1400/4768 loss 3.3268 lr 6.00e-04 gnorm 0.09 54,382 tok/s peakVRAM 13.7GB +[08:37:31] step 1420/4768 loss 3.3143 lr 6.00e-04 gnorm 0.10 54,385 tok/s peakVRAM 13.7GB +[08:40:44] step 1440/4768 loss 3.3218 lr 6.00e-04 gnorm 0.08 54,383 tok/s peakVRAM 13.7GB +[08:43:56] step 1460/4768 loss 3.3195 lr 6.00e-04 gnorm 0.09 54,381 tok/s peakVRAM 13.7GB +[08:47:09] step 1480/4768 loss 3.3233 lr 6.00e-04 gnorm 0.08 54,379 tok/s peakVRAM 13.7GB +[08:50:22] step 1500/4768 loss 3.2963 lr 6.00e-04 gnorm 0.10 54,381 tok/s peakVRAM 13.7GB +[08:50:24] ckpt @ 1500 +[08:53:37] step 1520/4768 loss 3.3041 lr 6.00e-04 gnorm 0.09 53,713 tok/s peakVRAM 13.7GB +[08:56:50] step 1540/4768 loss 3.2883 lr 6.00e-04 gnorm 0.09 54,374 tok/s peakVRAM 13.7GB +[09:00:03] step 1560/4768 loss 3.2969 lr 6.00e-04 gnorm 0.08 54,377 tok/s peakVRAM 13.7GB +[09:03:16] step 1580/4768 loss 3.2851 lr 6.00e-04 gnorm 0.09 54,376 tok/s peakVRAM 13.7GB +[09:06:29] step 1600/4768 loss 3.2750 lr 6.00e-04 gnorm 0.09 54,378 tok/s peakVRAM 13.7GB +[09:09:41] step 1620/4768 loss 3.2753 lr 6.00e-04 gnorm 0.09 54,376 tok/s peakVRAM 13.7GB +[09:13:59] step 1640/4768 loss 3.2726 lr 6.00e-04 gnorm 0.09 40,752 tok/s peakVRAM 13.7GB +[09:18:26] step 1660/4768 loss 3.2733 lr 6.00e-04 gnorm 0.09 39,290 tok/s peakVRAM 13.7GB +[09:22:46] step 1680/4768 loss 3.2623 lr 6.00e-04 gnorm 0.10 40,313 tok/s peakVRAM 13.7GB +[09:25:59] step 1700/4768 loss 3.2555 lr 6.00e-04 gnorm 0.09 54,380 tok/s peakVRAM 13.7GB +[09:29:11] step 1720/4768 loss 3.2578 lr 6.00e-04 gnorm 0.09 54,380 tok/s peakVRAM 13.7GB +[09:32:24] step 1740/4768 loss 3.2513 lr 6.00e-04 gnorm 0.10 54,385 tok/s peakVRAM 13.7GB +[09:35:37] step 1760/4768 loss 3.2508 lr 6.00e-04 gnorm 0.10 54,389 tok/s peakVRAM 13.7GB +[09:38:50] step 1780/4768 loss 3.2381 lr 6.00e-04 gnorm 0.10 54,384 tok/s peakVRAM 13.7GB +[09:42:03] step 1800/4768 loss 3.2418 lr 6.00e-04 gnorm 0.10 54,379 tok/s peakVRAM 13.7GB +[09:45:18] step 1820/4768 loss 3.2374 lr 6.00e-04 gnorm 0.08 53,646 tok/s peakVRAM 13.7GB +[09:51:02] step 1840/4768 loss 3.2313 lr 6.00e-04 gnorm 0.09 30,467 tok/s peakVRAM 13.7GB +[09:54:58] step 1860/4768 loss 3.2303 lr 6.00e-04 gnorm 0.09 44,403 tok/s peakVRAM 13.7GB +[10:00:30] step 1880/4768 loss 3.2265 lr 6.00e-04 gnorm 0.08 31,623 tok/s peakVRAM 13.7GB +[10:06:04] step 1900/4768 loss 3.2209 lr 6.00e-04 gnorm 0.09 31,365 tok/s peakVRAM 13.7GB +[10:11:58] step 1920/4768 loss 3.2205 lr 6.00e-04 gnorm 0.08 29,665 tok/s peakVRAM 13.7GB +[10:17:51] step 1940/4768 loss 3.2141 lr 6.00e-04 gnorm 0.08 29,662 tok/s peakVRAM 13.7GB +[10:23:36] step 1960/4768 loss 3.2180 lr 6.00e-04 gnorm 0.08 30,457 tok/s peakVRAM 13.7GB +[10:29:02] step 1980/4768 loss 3.2126 lr 6.00e-04 gnorm 0.09 32,076 tok/s peakVRAM 13.7GB +[10:34:30] step 2000/4768 loss 3.2058 lr 6.00e-04 gnorm 0.09 32,047 tok/s peakVRAM 13.7GB +[10:34:32] ckpt @ 2000 +[10:40:00] step 2020/4768 loss 3.2055 lr 6.00e-04 gnorm 0.09 31,786 tok/s peakVRAM 13.7GB +[10:45:28] step 2040/4768 loss 3.1983 lr 6.00e-04 gnorm 0.09 31,965 tok/s peakVRAM 13.7GB +[10:50:55] step 2060/4768 loss 3.1986 lr 6.00e-04 gnorm 0.08 31,995 tok/s peakVRAM 13.7GB +[10:56:23] step 2080/4768 loss 3.1876 lr 6.00e-04 gnorm 0.08 32,010 tok/s peakVRAM 13.7GB +[11:01:51] step 2100/4768 loss 3.1961 lr 6.00e-04 gnorm 0.10 31,946 tok/s peakVRAM 13.7GB +[11:07:19] step 2120/4768 loss 3.1859 lr 6.00e-04 gnorm 0.09 31,941 tok/s peakVRAM 13.7GB +[11:12:47] step 2140/4768 loss 3.1989 lr 6.00e-04 gnorm 0.08 32,013 tok/s peakVRAM 13.7GB +[11:18:15] step 2160/4768 loss 3.1776 lr 6.00e-04 gnorm 0.08 31,944 tok/s peakVRAM 13.7GB +[11:23:44] step 2180/4768 loss 3.1793 lr 6.00e-04 gnorm 0.09 31,909 tok/s peakVRAM 13.7GB +[11:28:11] step 2200/4768 loss 3.1803 lr 6.00e-04 gnorm 0.08 39,315 tok/s peakVRAM 13.7GB +[11:33:16] step 2220/4768 loss 3.1734 lr 6.00e-04 gnorm 0.09 34,363 tok/s peakVRAM 13.7GB +[11:39:24] step 2240/4768 loss 3.1700 lr 6.00e-04 gnorm 0.08 28,500 tok/s peakVRAM 13.7GB +[11:44:46] step 2260/4768 loss 3.1662 lr 6.00e-04 gnorm 0.08 32,518 tok/s peakVRAM 13.7GB +[11:50:32] step 2280/4768 loss 3.1625 lr 6.00e-04 gnorm 0.08 30,346 tok/s peakVRAM 13.7GB +[11:56:29] step 2300/4768 loss 3.1659 lr 6.00e-04 gnorm 0.08 29,311 tok/s peakVRAM 13.7GB +[12:01:34] step 2320/4768 loss 3.1626 lr 6.00e-04 gnorm 0.08 34,480 tok/s peakVRAM 13.7GB +[12:07:48] step 2340/4768 loss 3.1578 lr 6.00e-04 gnorm 0.09 28,010 tok/s peakVRAM 13.7GB diff --git a/R1/run/args.txt b/R1/run/args.txt new file mode 100644 index 0000000000000000000000000000000000000000..e705fc6de422919a4ba3dc415b376e249f4f48e1 --- /dev/null +++ b/R1/run/args.txt @@ -0,0 +1 @@ +--data-dir /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2 --out-dir /home/ubuntu/kolektyw/dowody400m/wyniki/R1 --run-id dowody_R1 --vocab 32768 --dtype uint16 --block 4096 --n-layer 16 --n-embd 896 --n-head 14 --value-residual --batch 128 --grad-accum 8 --norm rmsnorm --pos rope --rope-theta 500000 --ffn swiglu --ffn-mult 2.667 --qk-norm --attn-bias 0 --optimizer muon --muon-lr 0.02 --lr 6e-4 --min-lr 6e-05 --wd 0.1 --schedule wsd --decay-frac 0.2 --warmup 500 --act-ckpt --loss-chunk 1024 --sampler random --seed 1 --data-seed 20261022 --compile --log-every 20 --eval-every 4769 --eval-iters 1 --steps 4768 --ckpt-every 4768 \ No newline at end of file diff --git a/R1/run/metrics.jsonl b/R1/run/metrics.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..110c15101e6828f8f71d6d9c49f68adaf011d89a --- /dev/null +++ b/R1/run/metrics.jsonl @@ -0,0 +1,238 @@ +{"step": 20, "loss": 10.375725317001343, "lr": 2.3999999999999997e-05, "tok_s": 40192.52482876788} +{"step": 40, "loss": 9.55638346672058, "lr": 4.7999999999999994e-05, "tok_s": 54573.36660410555} +{"step": 60, "loss": 8.694436502456664, "lr": 7.199999999999999e-05, "tok_s": 54576.837488206365} +{"step": 80, "loss": 8.043932247161866, "lr": 9.599999999999999e-05, "tok_s": 54574.84322672207} +{"step": 100, "loss": 7.589998555183411, "lr": 0.00011999999999999999, "tok_s": 54563.929551511224} +{"step": 120, "loss": 7.120315837860107, "lr": 0.00014399999999999998, "tok_s": 54555.47091571738} +{"step": 140, "loss": 6.705103468894959, "lr": 0.000168, "tok_s": 54545.749158098115} +{"step": 160, "loss": 6.349775433540344, "lr": 0.00019199999999999998, "tok_s": 54543.69310894704} +{"step": 180, "loss": 6.0849058628082275, "lr": 0.00021599999999999996, "tok_s": 54537.1582642568} +{"step": 200, "loss": 5.863923573493958, "lr": 0.00023999999999999998, "tok_s": 54535.132210957454} +{"step": 220, "loss": 5.675646543502808, "lr": 0.00026399999999999997, "tok_s": 54530.308564857034} +{"step": 240, "loss": 5.4996884107589725, "lr": 0.00028799999999999995, "tok_s": 54529.76869794185} +{"step": 260, "loss": 5.3284769535064695, "lr": 0.000312, "tok_s": 54527.51935625258} +{"step": 280, "loss": 5.1731541633605955, "lr": 0.000336, "tok_s": 54528.11529102413} +{"step": 300, "loss": 5.028257584571838, "lr": 0.00035999999999999997, "tok_s": 54530.33195820483} +{"step": 320, "loss": 4.893511152267456, "lr": 0.00038399999999999996, "tok_s": 54529.89418139586} +{"step": 340, "loss": 4.750537443161011, "lr": 0.000408, "tok_s": 54526.910184437875} +{"step": 360, "loss": 4.624461531639099, "lr": 0.00043199999999999993, "tok_s": 54527.126715707716} +{"step": 380, "loss": 4.506671214103699, "lr": 0.00045599999999999997, "tok_s": 54527.24461544256} +{"step": 400, "loss": 4.402042651176453, "lr": 0.00047999999999999996, "tok_s": 54527.53902899062} +{"step": 420, "loss": 4.325919914245605, "lr": 0.000504, "tok_s": 54523.748537048334} +{"step": 440, "loss": 4.247882843017578, "lr": 0.0005279999999999999, "tok_s": 54526.16122800285} +{"step": 460, "loss": 4.1796149730682375, "lr": 0.000552, "tok_s": 54526.823585931896} +{"step": 480, "loss": 4.129853081703186, "lr": 0.0005759999999999999, "tok_s": 54524.81675443662} +{"step": 500, "loss": 4.077749156951905, "lr": 0.0006, "tok_s": 54522.929438760264} +{"step": 520, "loss": 4.039146566390992, "lr": 0.0006, "tok_s": 54519.68458328312} +{"step": 540, "loss": 3.9884557843208315, "lr": 0.0006, "tok_s": 54519.765887510344} +{"step": 560, "loss": 3.936927056312561, "lr": 0.0006, "tok_s": 54520.50770685465} +{"step": 580, "loss": 3.896349775791168, "lr": 0.0006, "tok_s": 54518.879799407354} +{"step": 600, "loss": 3.8545740485191344, "lr": 0.0006, "tok_s": 54522.30002461412} +{"step": 620, "loss": 3.832847225666046, "lr": 0.0006, "tok_s": 54520.07083121125} +{"step": 640, "loss": 3.8063215017318726, "lr": 0.0006, "tok_s": 54519.17905620606} +{"step": 660, "loss": 3.7745464205741883, "lr": 0.0006, "tok_s": 54520.43511904895} +{"step": 680, "loss": 3.753047776222229, "lr": 0.0006, "tok_s": 54519.251573083806} +{"step": 700, "loss": 3.7221386671066283, "lr": 0.0006, "tok_s": 54518.82154338457} +{"step": 720, "loss": 3.6911539673805236, "lr": 0.0006, "tok_s": 54516.745966587485} +{"step": 740, "loss": 3.6872825026512146, "lr": 0.0006, "tok_s": 54513.32772194921} +{"step": 760, "loss": 3.6627037644386293, "lr": 0.0006, "tok_s": 54512.705421316496} +{"step": 780, "loss": 3.6369429230690002, "lr": 0.0006, "tok_s": 54511.63950266863} +{"step": 800, "loss": 3.623213994503021, "lr": 0.0006, "tok_s": 54508.2637155628} +{"step": 820, "loss": 3.607566463947296, "lr": 0.0006, "tok_s": 54511.56254679347} +{"step": 840, "loss": 3.5929282784461973, "lr": 0.0006, "tok_s": 54509.75944972368} +{"step": 860, "loss": 3.5715310096740724, "lr": 0.0006, "tok_s": 54509.64851666642} +{"step": 880, "loss": 3.5661336302757265, "lr": 0.0006, "tok_s": 54508.117659562035} +{"step": 900, "loss": 3.547942078113556, "lr": 0.0006, "tok_s": 54510.204267443245} +{"step": 920, "loss": 3.54037184715271, "lr": 0.0006, "tok_s": 54510.82698390962} +{"step": 940, "loss": 3.534908449649811, "lr": 0.0006, "tok_s": 54510.50660440851} +{"step": 960, "loss": 3.5100515842437745, "lr": 0.0006, "tok_s": 54510.87508845343} +{"step": 980, "loss": 3.5061208605766296, "lr": 0.0006, "tok_s": 54509.29092615169} +{"step": 1000, "loss": 3.4963834166526793, "lr": 0.0006, "tok_s": 54509.491373534765} +{"step": 1020, "loss": 3.4905299305915833, "lr": 0.0006, "tok_s": 54506.1150418635} +{"step": 1040, "loss": 3.4632843494415284, "lr": 0.0006, "tok_s": 54504.96055521561} +{"step": 1060, "loss": 3.4752270936965943, "lr": 0.0006, "tok_s": 54505.576937326645} +{"step": 1080, "loss": 3.4639549970626833, "lr": 0.0006, "tok_s": 54502.19041905023} +{"step": 1100, "loss": 3.448479986190796, "lr": 0.0006, "tok_s": 54501.500968276836} +{"step": 1120, "loss": 3.430601716041565, "lr": 0.0006, "tok_s": 54497.39305969334} +{"step": 1140, "loss": 3.423946189880371, "lr": 0.0006, "tok_s": 54497.005647397564} +{"step": 1160, "loss": 3.418174147605896, "lr": 0.0006, "tok_s": 54502.22560798331} +{"step": 1180, "loss": 3.420488750934601, "lr": 0.0006, "tok_s": 54502.96100629495} +{"step": 1200, "loss": 3.4019625663757322, "lr": 0.0006, "tok_s": 54501.15395312126} +{"step": 1220, "loss": 3.399947392940521, "lr": 0.0006, "tok_s": 54503.564914990726} +{"step": 1240, "loss": 3.390988624095917, "lr": 0.0006, "tok_s": 54502.225405359764} +{"step": 1260, "loss": 3.3768735408782957, "lr": 0.0006, "tok_s": 54503.942086096744} +{"step": 1280, "loss": 3.380297088623047, "lr": 0.0006, "tok_s": 54503.82739418731} +{"step": 1300, "loss": 3.3731743574142454, "lr": 0.0006, "tok_s": 54505.2235882436} +{"step": 1320, "loss": 3.3656145334243774, "lr": 0.0006, "tok_s": 54502.43829599525} +{"step": 1340, "loss": 3.3656646490097044, "lr": 0.0006, "tok_s": 54499.28522618628} +{"step": 1360, "loss": 3.35660183429718, "lr": 0.0006, "tok_s": 54498.06511863051} +{"step": 1380, "loss": 3.347859597206116, "lr": 0.0006, "tok_s": 54498.61644623614} +{"step": 1400, "loss": 3.337520945072174, "lr": 0.0006, "tok_s": 54504.5342633703} +{"step": 1420, "loss": 3.3259835839271545, "lr": 0.0006, "tok_s": 54506.20312827794} +{"step": 1440, "loss": 3.332594668865204, "lr": 0.0006, "tok_s": 54506.02702328437} +{"step": 1460, "loss": 3.32985098361969, "lr": 0.0006, "tok_s": 54503.571872075256} +{"step": 1480, "loss": 3.33380606174469, "lr": 0.0006, "tok_s": 54505.17745258027} +{"step": 1500, "loss": 3.3074196696281435, "lr": 0.0006, "tok_s": 54506.83271124733} +{"step": 1520, "loss": 3.314147639274597, "lr": 0.0006, "tok_s": 54505.6908935518} +{"step": 1540, "loss": 3.29811692237854, "lr": 0.0006, "tok_s": 54504.80614101511} +{"step": 1560, "loss": 3.306923580169678, "lr": 0.0006, "tok_s": 54501.88405436665} +{"step": 1580, "loss": 3.2949591875076294, "lr": 0.0006, "tok_s": 54497.369221894354} +{"step": 1600, "loss": 3.284385287761688, "lr": 0.0006, "tok_s": 54499.40327568954} +{"step": 1620, "loss": 3.2848113894462587, "lr": 0.0006, "tok_s": 54498.42040085727} +{"step": 1640, "loss": 3.2816879749298096, "lr": 0.0006, "tok_s": 54500.64674351206} +{"step": 1660, "loss": 3.282590997219086, "lr": 0.0006, "tok_s": 54495.714808041674} +{"step": 1680, "loss": 3.271337866783142, "lr": 0.0006, "tok_s": 54497.035697483174} +{"step": 1700, "loss": 3.2643879294395446, "lr": 0.0006, "tok_s": 54496.460429801904} +{"step": 1720, "loss": 3.2666760683059692, "lr": 0.0006, "tok_s": 54494.41538704} +{"step": 1740, "loss": 3.2595957279205323, "lr": 0.0006, "tok_s": 54492.01698230334} +{"step": 1760, "loss": 3.2589515805244447, "lr": 0.0006, "tok_s": 54496.70481072665} +{"step": 1780, "loss": 3.246141254901886, "lr": 0.0006, "tok_s": 54500.78195345969} +{"step": 1800, "loss": 3.250395131111145, "lr": 0.0006, "tok_s": 54497.50347016742} +{"step": 1820, "loss": 3.2456166625022886, "lr": 0.0006, "tok_s": 54502.275318338565} +{"step": 1840, "loss": 3.2389777541160583, "lr": 0.0006, "tok_s": 54501.12518172441} +{"step": 1860, "loss": 3.2379404306411743, "lr": 0.0006, "tok_s": 54499.549622665865} +{"step": 1880, "loss": 3.2338213324546814, "lr": 0.0006, "tok_s": 54502.01035508657} +{"step": 1900, "loss": 3.2283488750457763, "lr": 0.0006, "tok_s": 54500.42542478084} +{"step": 1920, "loss": 3.2282689452171325, "lr": 0.0006, "tok_s": 54502.37048420369} +{"step": 1940, "loss": 3.220784342288971, "lr": 0.0006, "tok_s": 54499.72359222135} +{"step": 1960, "loss": 3.2250001192092896, "lr": 0.0006, "tok_s": 54499.94551307226} +{"step": 1980, "loss": 3.2195660352706907, "lr": 0.0006, "tok_s": 54502.737642494045} +{"step": 2000, "loss": 3.2127142429351805, "lr": 0.0006, "tok_s": 54502.04892082413} +{"step": 2020, "loss": 3.2123374581336974, "lr": 0.0006, "tok_s": 54503.26907168031} +{"step": 2040, "loss": 3.2056957364082335, "lr": 0.0006, "tok_s": 54505.24831104701} +{"step": 2060, "loss": 3.206089961528778, "lr": 0.0006, "tok_s": 54507.531619419016} +{"step": 2080, "loss": 3.1947311878204347, "lr": 0.0006, "tok_s": 54507.16730133403} +{"step": 2100, "loss": 3.203057360649109, "lr": 0.0006, "tok_s": 54506.94586208892} +{"step": 2120, "loss": 3.192316246032715, "lr": 0.0006, "tok_s": 54510.03712214494} +{"step": 2140, "loss": 3.205461084842682, "lr": 0.0006, "tok_s": 54509.02785386576} +{"step": 2160, "loss": 3.184077429771423, "lr": 0.0006, "tok_s": 54505.62564055129} +{"step": 2180, "loss": 3.1858996033668516, "lr": 0.0006, "tok_s": 54503.09062162907} +{"step": 2200, "loss": 3.187498140335083, "lr": 0.0006, "tok_s": 54505.14435383624} +{"step": 2220, "loss": 3.179544746875763, "lr": 0.0006, "tok_s": 54505.3278159521} +{"step": 2240, "loss": 3.1762046694755552, "lr": 0.0006, "tok_s": 54505.814442362724} +{"step": 2260, "loss": 3.1724456191062926, "lr": 0.0006, "tok_s": 54505.295595159725} +{"step": 2280, "loss": 3.168820869922638, "lr": 0.0006, "tok_s": 54506.82048422928} +{"step": 2300, "loss": 3.172169578075409, "lr": 0.0006, "tok_s": 54505.36895324553} +{"step": 2320, "loss": 3.169022965431213, "lr": 0.0006, "tok_s": 54506.215760325445} +{"step": 2340, "loss": 3.1635345220565796, "lr": 0.0006, "tok_s": 54503.63495872872} +{"step": 2360, "loss": 3.154193341732025, "lr": 0.0006, "tok_s": 54503.75883594428} +{"step": 2380, "loss": 3.1532516360282896, "lr": 0.0006, "tok_s": 54501.41316721653} +{"step": 2400, "loss": 3.153403079509735, "lr": 0.0006, "tok_s": 54502.95931771989} +{"step": 2420, "loss": 3.1562620282173155, "lr": 0.0006, "tok_s": 54497.84503641175} +{"step": 2440, "loss": 3.150172305107117, "lr": 0.0006, "tok_s": 54499.854137652146} +{"step": 2460, "loss": 3.148875081539154, "lr": 0.0006, "tok_s": 54499.04176758424} +{"step": 2480, "loss": 3.146116590499878, "lr": 0.0006, "tok_s": 54499.657880764156} +{"step": 2500, "loss": 3.140621912479401, "lr": 0.0006, "tok_s": 54499.554417619314} +{"step": 2520, "loss": 3.1425031423568726, "lr": 0.0006, "tok_s": 54504.1037903509} +{"step": 2540, "loss": 3.1422348022460938, "lr": 0.0006, "tok_s": 54503.31284007723} +{"step": 2560, "loss": 3.1397387862205504, "lr": 0.0006, "tok_s": 54503.85177799809} +{"step": 2580, "loss": 3.1356419563293456, "lr": 0.0006, "tok_s": 54502.07154698277} +{"step": 2600, "loss": 3.125533878803253, "lr": 0.0006, "tok_s": 54505.3636168772} +{"step": 2620, "loss": 3.1342557787895204, "lr": 0.0006, "tok_s": 54503.0855558801} +{"step": 2640, "loss": 3.1347974300384522, "lr": 0.0006, "tok_s": 54501.72979267718} +{"step": 2660, "loss": 3.133404326438904, "lr": 0.0006, "tok_s": 54498.91284683648} +{"step": 2680, "loss": 3.1281450986862183, "lr": 0.0006, "tok_s": 54501.83981548286} +{"step": 2700, "loss": 3.1312366247177126, "lr": 0.0006, "tok_s": 54503.26940939915} +{"step": 2720, "loss": 3.1138309240341187, "lr": 0.0006, "tok_s": 54499.216814437044} +{"step": 2740, "loss": 3.1155006527900695, "lr": 0.0006, "tok_s": 54501.64347675383} +{"step": 2760, "loss": 3.118541395664215, "lr": 0.0006, "tok_s": 54500.04836987996} +{"step": 2780, "loss": 3.1097805500030518, "lr": 0.0006, "tok_s": 54502.48861461497} +{"step": 2800, "loss": 3.1045394062995912, "lr": 0.0006, "tok_s": 54499.85366490502} +{"step": 2820, "loss": 3.1016828536987306, "lr": 0.0006, "tok_s": 54501.25593648621} +{"step": 2840, "loss": 3.109878695011139, "lr": 0.0006, "tok_s": 54503.062658706505} +{"step": 2860, "loss": 3.1015298843383787, "lr": 0.0006, "tok_s": 54503.35221818577} +{"step": 2880, "loss": 3.1036748051643372, "lr": 0.0006, "tok_s": 54501.39317563004} +{"step": 2900, "loss": 3.1047995686531067, "lr": 0.0006, "tok_s": 54496.90557072221} +{"step": 2920, "loss": 3.1120428085327148, "lr": 0.0006, "tok_s": 54499.5828496846} +{"step": 2940, "loss": 3.1023404002189636, "lr": 0.0006, "tok_s": 54504.77317781704} +{"step": 2960, "loss": 3.1040467023849487, "lr": 0.0006, "tok_s": 54499.708802064546} +{"step": 2980, "loss": 3.091260254383087, "lr": 0.0006, "tok_s": 54500.4839116434} +{"step": 3000, "loss": 3.0920066237449646, "lr": 0.0006, "tok_s": 54501.659078438046} +{"step": 3020, "loss": 3.0873806476593018, "lr": 0.0006, "tok_s": 54504.41321958712} +{"step": 3040, "loss": 3.084467089176178, "lr": 0.0006, "tok_s": 54500.721169683944} +{"step": 3060, "loss": 3.0876251578330995, "lr": 0.0006, "tok_s": 54502.8793469247} +{"step": 3080, "loss": 3.0809731602668764, "lr": 0.0006, "tok_s": 54507.358410658584} +{"step": 3100, "loss": 3.0741262912750242, "lr": 0.0006, "tok_s": 54507.26741570171} +{"step": 3120, "loss": 3.090480697154999, "lr": 0.0006, "tok_s": 54506.95032057946} +{"step": 3140, "loss": 3.076385426521301, "lr": 0.0006, "tok_s": 54505.714738631345} +{"step": 3160, "loss": 3.0722206592559815, "lr": 0.0006, "tok_s": 54504.217537777884} +{"step": 3180, "loss": 3.0782618522644043, "lr": 0.0006, "tok_s": 54504.55418971427} +{"step": 3200, "loss": 3.0793707728385926, "lr": 0.0006, "tok_s": 54506.32255877922} +{"step": 3220, "loss": 3.08889821767807, "lr": 0.0006, "tok_s": 54507.27883227577} +{"step": 3240, "loss": 3.07084139585495, "lr": 0.0006, "tok_s": 54504.121487365956} +{"step": 3260, "loss": 3.0676586627960205, "lr": 0.0006, "tok_s": 54503.06833234065} +{"step": 3280, "loss": 3.0627394318580627, "lr": 0.0006, "tok_s": 54503.69372265935} +{"step": 3300, "loss": 3.0739628314971923, "lr": 0.0006, "tok_s": 54506.213868895495} +{"step": 3320, "loss": 3.0676796793937684, "lr": 0.0006, "tok_s": 54504.716032695185} +{"step": 3340, "loss": 3.0680676341056823, "lr": 0.0006, "tok_s": 54504.37701463143} +{"step": 3360, "loss": 3.0593183636665344, "lr": 0.0006, "tok_s": 54507.069957014726} +{"step": 3380, "loss": 3.0686938881874086, "lr": 0.0006, "tok_s": 54504.56479458801} +{"step": 3400, "loss": 3.0537588715553285, "lr": 0.0006, "tok_s": 54503.721483594796} +{"step": 3420, "loss": 3.061671257019043, "lr": 0.0006, "tok_s": 54503.8769048287} +{"step": 3440, "loss": 3.053264784812927, "lr": 0.0006, "tok_s": 54503.23401648718} +{"step": 3460, "loss": 3.0629284143447877, "lr": 0.0006, "tok_s": 54505.903405959434} +{"step": 3480, "loss": 3.0617974996566772, "lr": 0.0006, "tok_s": 54506.13415875287} +{"step": 3500, "loss": 3.0478788137435915, "lr": 0.0006, "tok_s": 54505.478720547515} +{"step": 3520, "loss": 3.0510458111763, "lr": 0.0006, "tok_s": 54505.30241758848} +{"step": 3540, "loss": 3.05376158952713, "lr": 0.0006, "tok_s": 54505.42245208456} +{"step": 3560, "loss": 3.058077025413513, "lr": 0.0006, "tok_s": 54503.38673317847} +{"step": 3580, "loss": 3.0513238310813904, "lr": 0.0006, "tok_s": 54502.17792395057} +{"step": 3600, "loss": 3.05366827249527, "lr": 0.0006, "tok_s": 54505.76735999967} +{"step": 3620, "loss": 3.0450515270233156, "lr": 0.0006, "tok_s": 54506.02736103739} +{"step": 3640, "loss": 3.053074300289154, "lr": 0.0006, "tok_s": 54505.65759153513} +{"step": 3660, "loss": 3.0417879939079286, "lr": 0.0006, "tok_s": 54507.56107312429} +{"step": 3680, "loss": 3.0484514594078065, "lr": 0.0006, "tok_s": 54510.41316672687} +{"step": 3700, "loss": 3.0457404017448426, "lr": 0.0006, "tok_s": 54510.74732783625} +{"step": 3720, "loss": 3.039131832122803, "lr": 0.0006, "tok_s": 54509.28234619887} +{"step": 3740, "loss": 3.0350921869277956, "lr": 0.0006, "tok_s": 54507.17371891067} +{"step": 3760, "loss": 3.0340383052825928, "lr": 0.0006, "tok_s": 54508.444969310796} +{"step": 3780, "loss": 3.034581756591797, "lr": 0.0006, "tok_s": 54507.59951160865} +{"step": 3800, "loss": 3.0352933168411256, "lr": 0.0006, "tok_s": 54509.21134212601} +{"step": 3820, "loss": 3.0289077758789062, "lr": 0.0005609064981513237, "tok_s": 54503.22158845265} +{"step": 3840, "loss": 3.0271220803260803, "lr": 0.0005125842723878462, "tok_s": 54505.45001213353} +{"step": 3860, "loss": 3.0203971028327943, "lr": 0.0004827194944539712, "tok_s": 54505.29174487891} +{"step": 3880, "loss": 3.017254042625427, "lr": 0.0004590463745471515, "tok_s": 54507.10880012612} +{"step": 3900, "loss": 3.010846734046936, "lr": 0.0004388133626026285, "tok_s": 54502.2665379703} +{"step": 3920, "loss": 3.005240833759308, "lr": 0.000420851068597545, "tok_s": 54503.841376037366} +{"step": 3940, "loss": 2.997415769100189, "lr": 0.00040453249075661876, "tok_s": 54506.9768013262} +{"step": 3960, "loss": 2.9981491565704346, "lr": 0.00038947504965686095, "tok_s": 54505.679274993694} +{"step": 3980, "loss": 2.989938259124756, "lr": 0.000375424929557729, "tok_s": 54503.48575295003} +{"step": 4000, "loss": 2.989367723464966, "lr": 0.00036220351177672794, "tok_s": 54504.00699733829} +{"step": 4020, "loss": 2.9862194895744323, "lr": 0.0003496794507866152, "tok_s": 54505.02763042305} +{"step": 4040, "loss": 2.9767078161239624, "lr": 0.0003377528171635386, "tok_s": 54503.98990830757} +{"step": 4060, "loss": 2.97679283618927, "lr": 0.00032634548705926626, "tok_s": 54504.066775257335} +{"step": 4080, "loss": 2.9677639603614807, "lr": 0.00031539501058484577, "tok_s": 54505.65819948233} +{"step": 4100, "loss": 2.973319184780121, "lr": 0.00030485053354754334, "tok_s": 54502.66665547048} +{"step": 4120, "loss": 2.9620593547821046, "lr": 0.00029466998985632013, "tok_s": 54504.37498823608} +{"step": 4140, "loss": 2.9543420553207396, "lr": 0.0002848181118123862, "tok_s": 54505.295054769405} +{"step": 4160, "loss": 2.972287619113922, "lr": 0.000275264984658754, "tok_s": 54506.23305340531} +{"step": 4180, "loss": 2.964927887916565, "lr": 0.0002659849737873323, "tok_s": 54507.81339000021} +{"step": 4200, "loss": 2.9585041284561155, "lr": 0.0002569559134994179, "tok_s": 54509.20701838254} +{"step": 4220, "loss": 2.9517848968505858, "lr": 0.00024815848336191376, "tok_s": 54509.775191174595} +{"step": 4240, "loss": 2.9586602091789245, "lr": 0.00023957572171486765, "tok_s": 54507.26390291066} +{"step": 4260, "loss": 2.9503562092781066, "lr": 0.0002311926411735683, "tok_s": 54510.10360180129} +{"step": 4280, "loss": 2.949582040309906, "lr": 0.0002229959211471338, "tok_s": 54512.47082915212} +{"step": 4300, "loss": 2.947797930240631, "lr": 0.00021497365931813446, "tok_s": 54509.605548806976} +{"step": 4320, "loss": 2.9426231741905213, "lr": 0.00020711516882674415, "tok_s": 54508.71958826259} +{"step": 4340, "loss": 2.9411731123924256, "lr": 0.00019941081128766396, "tok_s": 54506.814877366494} +{"step": 4360, "loss": 2.9349792838096618, "lr": 0.00019185185819315875, "tok_s": 54504.90469310944} +{"step": 4380, "loss": 2.942066013813019, "lr": 0.00018443037501815989, "tok_s": 54508.95245919496} +{"step": 4400, "loss": 2.9398043632507322, "lr": 0.00017713912364145464, "tok_s": 54509.180738144045} +{"step": 4420, "loss": 2.932943570613861, "lr": 0.00016997147966456131, "tok_s": 54506.15124912823} +{"step": 4440, "loss": 2.9391417503356934, "lr": 0.00016292136193923112, "tok_s": 54507.113326196624} +{"step": 4460, "loss": 2.9347523927688597, "lr": 0.00015598317216998812, "tok_s": 54504.91293394499} +{"step": 4480, "loss": 2.9299306750297545, "lr": 0.00014915174288526275, "tok_s": 54503.95809436984} +{"step": 4500, "loss": 2.927113926410675, "lr": 0.00014242229240208165, "tok_s": 54502.22655355987} +{"step": 4520, "loss": 2.929121160507202, "lr": 0.00013579038566856907, "tok_s": 54503.620436627454} +{"step": 4540, "loss": 2.920461642742157, "lr": 0.00012925190007297358, "tok_s": 54503.404632360194} +{"step": 4560, "loss": 2.9235119938850405, "lr": 0.0001228029954703451, "tok_s": 54504.438144295396} +{"step": 4580, "loss": 2.926887738704681, "lr": 0.00011644008780788571, "tok_s": 54503.777951180906} +{"step": 4600, "loss": 2.9252463817596435, "lr": 0.00011015982583457378, "tok_s": 54506.12341816673} +{"step": 4620, "loss": 2.9120384812355042, "lr": 0.00010395907046536478, "tok_s": 54506.67186897893} +{"step": 4640, "loss": 2.921280491352081, "lr": 9.783487643927846e-05, "tok_s": 54508.41929781589} +{"step": 4660, "loss": 2.9140223622322083, "lr": 9.178447596720887e-05, "tok_s": 54508.05962920361} +{"step": 4680, "loss": 2.9119210481643676, "lr": 8.580526411184025e-05, "tok_s": 54507.65997295472} +{"step": 4700, "loss": 2.9078417301177977, "lr": 7.98947856805662e-05, "tok_s": 54506.66004734608} +{"step": 4720, "loss": 2.90886070728302, "lr": 7.405072344432876e-05, "tok_s": 54505.05843238631} +{"step": 4740, "loss": 2.906625306606293, "lr": 6.827088752203553e-05, "tok_s": 54507.1871620498} +{"step": 4760, "loss": 2.9156633734703066, "lr": 6.255320579263513e-05, "tok_s": 54512.37562801159} diff --git a/R1/run/srodowisko.txt b/R1/run/srodowisko.txt new file mode 100644 index 0000000000000000000000000000000000000000..e7e2dd3ec25f8f96cde2b26a7eaf5cd13a428e7c --- /dev/null +++ b/R1/run/srodowisko.txt @@ -0,0 +1,63 @@ +data 2026-10-22T11:59:49Z ARM R1 GPU 0 MINI 0 DATA_SEED 20261022 +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +a607076bd6e11cad879b8213b37492a4b4f5cbf8fa96205b7ebaeace8a25213e /home/ubuntu/kolektyw/dowody400m/kod/ramie.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 diff --git a/R1/run/srodowisko_R1_mini0.txt b/R1/run/srodowisko_R1_mini0.txt new file mode 100644 index 0000000000000000000000000000000000000000..e7e2dd3ec25f8f96cde2b26a7eaf5cd13a428e7c --- /dev/null +++ b/R1/run/srodowisko_R1_mini0.txt @@ -0,0 +1,63 @@ +data 2026-10-22T11:59:49Z ARM R1 GPU 0 MINI 0 DATA_SEED 20261022 +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +a607076bd6e11cad879b8213b37492a4b4f5cbf8fa96205b7ebaeace8a25213e /home/ubuntu/kolektyw/dowody400m/kod/ramie.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 diff --git a/R1/run/srodowisko_R1_mini1.txt b/R1/run/srodowisko_R1_mini1.txt new file mode 100644 index 0000000000000000000000000000000000000000..0b12d2d65188eb478062eb0b197f8d961143b9e7 --- /dev/null +++ b/R1/run/srodowisko_R1_mini1.txt @@ -0,0 +1,63 @@ +data 2026-10-22T11:48:41Z ARM R1 GPU 0 MINI 1 DATA_SEED 20261022 +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +a607076bd6e11cad879b8213b37492a4b4f5cbf8fa96205b7ebaeace8a25213e /home/ubuntu/kolektyw/dowody400m/kod/ramie.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 diff --git a/R1/run/train.log b/R1/run/train.log new file mode 100644 index 0000000000000000000000000000000000000000..684af70519c6b1198dd6d413eb6bf8456b6dda58 --- /dev/null +++ b/R1/run/train.log @@ -0,0 +1,248 @@ +[13:59:56] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[13:59:56] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[13:59:58] model GPT-ref: 183.6M param (L16 d896 h14) +[13:59:59] optimizer=muon muon_lr=0.02 (mult=33.3x) +[13:59:59] r11 pamiec: act_ckpt=True loss_chunk=1024 +[13:59:59] r13: mixer_layout=AAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[13:59:59] torch.compile enabled +[13:59:59] START gpt-ref: steps=4768 (od 0) lr=0.0006->6e-05 +[14:04:20] step 20/4768 loss 10.3757 lr 2.40e-05 gnorm 2.72 40,193 tok/s peakVRAM 13.7GB +[14:07:32] step 40/4768 loss 9.5564 lr 4.80e-05 gnorm 1.29 54,573 tok/s peakVRAM 13.7GB +[14:10:45] step 60/4768 loss 8.6944 lr 7.20e-05 gnorm 0.71 54,577 tok/s peakVRAM 13.7GB +[14:13:57] step 80/4768 loss 8.0439 lr 9.60e-05 gnorm 1.18 54,575 tok/s peakVRAM 13.7GB +[14:17:09] step 100/4768 loss 7.5900 lr 1.20e-04 gnorm 0.92 54,564 tok/s peakVRAM 13.7GB +[14:20:21] step 120/4768 loss 7.1203 lr 1.44e-04 gnorm 0.82 54,555 tok/s peakVRAM 13.7GB +[14:23:33] step 140/4768 loss 6.7051 lr 1.68e-04 gnorm 0.74 54,546 tok/s peakVRAM 13.7GB +[14:26:46] step 160/4768 loss 6.3498 lr 1.92e-04 gnorm 0.75 54,544 tok/s peakVRAM 13.7GB +[14:29:58] step 180/4768 loss 6.0849 lr 2.16e-04 gnorm 0.78 54,537 tok/s peakVRAM 13.7GB +[14:33:10] step 200/4768 loss 5.8639 lr 2.40e-04 gnorm 0.87 54,535 tok/s peakVRAM 13.7GB +[14:36:22] step 220/4768 loss 5.6756 lr 2.64e-04 gnorm 0.87 54,530 tok/s peakVRAM 13.7GB +[14:39:35] step 240/4768 loss 5.4997 lr 2.88e-04 gnorm 0.77 54,530 tok/s peakVRAM 13.7GB +[14:42:47] step 260/4768 loss 5.3285 lr 3.12e-04 gnorm 0.84 54,528 tok/s peakVRAM 13.7GB +[14:45:59] step 280/4768 loss 5.1732 lr 3.36e-04 gnorm 0.91 54,528 tok/s peakVRAM 13.7GB +[14:49:12] step 300/4768 loss 5.0283 lr 3.60e-04 gnorm 0.78 54,530 tok/s peakVRAM 13.7GB +[14:52:24] step 320/4768 loss 4.8935 lr 3.84e-04 gnorm 0.63 54,530 tok/s peakVRAM 13.7GB +[14:55:36] step 340/4768 loss 4.7505 lr 4.08e-04 gnorm 0.59 54,527 tok/s peakVRAM 13.7GB +[14:58:49] step 360/4768 loss 4.6245 lr 4.32e-04 gnorm 0.72 54,527 tok/s peakVRAM 13.7GB +[15:02:01] step 380/4768 loss 4.5067 lr 4.56e-04 gnorm 0.46 54,527 tok/s peakVRAM 13.7GB +[15:05:13] step 400/4768 loss 4.4020 lr 4.80e-04 gnorm 0.52 54,528 tok/s peakVRAM 13.7GB +[15:08:25] step 420/4768 loss 4.3259 lr 5.04e-04 gnorm 0.47 54,524 tok/s peakVRAM 13.7GB +[15:11:38] step 440/4768 loss 4.2479 lr 5.28e-04 gnorm 0.40 54,526 tok/s peakVRAM 13.7GB +[15:14:50] step 460/4768 loss 4.1796 lr 5.52e-04 gnorm 0.41 54,527 tok/s peakVRAM 13.7GB +[15:18:02] step 480/4768 loss 4.1299 lr 5.76e-04 gnorm 0.42 54,525 tok/s peakVRAM 13.7GB +[15:21:15] step 500/4768 loss 4.0777 lr 6.00e-04 gnorm 0.36 54,523 tok/s peakVRAM 13.7GB +[15:24:27] step 520/4768 loss 4.0391 lr 6.00e-04 gnorm 0.31 54,520 tok/s peakVRAM 13.7GB +[15:27:39] step 540/4768 loss 3.9885 lr 6.00e-04 gnorm 0.26 54,520 tok/s peakVRAM 13.7GB +[15:30:52] step 560/4768 loss 3.9369 lr 6.00e-04 gnorm 0.20 54,521 tok/s peakVRAM 13.7GB +[15:34:04] step 580/4768 loss 3.8963 lr 6.00e-04 gnorm 0.22 54,519 tok/s peakVRAM 13.7GB +[15:37:16] step 600/4768 loss 3.8546 lr 6.00e-04 gnorm 0.23 54,522 tok/s peakVRAM 13.7GB +[15:40:29] step 620/4768 loss 3.8328 lr 6.00e-04 gnorm 0.21 54,520 tok/s peakVRAM 13.7GB +[15:43:41] step 640/4768 loss 3.8063 lr 6.00e-04 gnorm 0.19 54,519 tok/s peakVRAM 13.7GB +[15:46:53] step 660/4768 loss 3.7745 lr 6.00e-04 gnorm 0.20 54,520 tok/s peakVRAM 13.7GB +[15:50:06] step 680/4768 loss 3.7530 lr 6.00e-04 gnorm 0.17 54,519 tok/s peakVRAM 13.7GB +[15:53:18] step 700/4768 loss 3.7221 lr 6.00e-04 gnorm 0.19 54,519 tok/s peakVRAM 13.7GB +[15:56:30] step 720/4768 loss 3.6912 lr 6.00e-04 gnorm 0.16 54,517 tok/s peakVRAM 13.7GB +[15:59:43] step 740/4768 loss 3.6873 lr 6.00e-04 gnorm 0.18 54,513 tok/s peakVRAM 13.7GB +[16:02:55] step 760/4768 loss 3.6627 lr 6.00e-04 gnorm 0.17 54,513 tok/s peakVRAM 13.7GB +[16:06:07] step 780/4768 loss 3.6369 lr 6.00e-04 gnorm 0.15 54,512 tok/s peakVRAM 13.7GB +[16:09:20] step 800/4768 loss 3.6232 lr 6.00e-04 gnorm 0.16 54,508 tok/s peakVRAM 13.7GB +[16:12:32] step 820/4768 loss 3.6076 lr 6.00e-04 gnorm 0.15 54,512 tok/s peakVRAM 13.7GB +[16:15:45] step 840/4768 loss 3.5929 lr 6.00e-04 gnorm 0.15 54,510 tok/s peakVRAM 13.7GB +[16:18:57] step 860/4768 loss 3.5715 lr 6.00e-04 gnorm 0.13 54,510 tok/s peakVRAM 13.7GB +[16:22:09] step 880/4768 loss 3.5661 lr 6.00e-04 gnorm 0.15 54,508 tok/s peakVRAM 13.7GB +[16:25:22] step 900/4768 loss 3.5479 lr 6.00e-04 gnorm 0.14 54,510 tok/s peakVRAM 13.7GB +[16:28:34] step 920/4768 loss 3.5404 lr 6.00e-04 gnorm 0.13 54,511 tok/s peakVRAM 13.7GB +[16:31:46] step 940/4768 loss 3.5349 lr 6.00e-04 gnorm 0.13 54,511 tok/s peakVRAM 13.7GB +[16:34:59] step 960/4768 loss 3.5101 lr 6.00e-04 gnorm 0.14 54,511 tok/s peakVRAM 13.7GB +[16:38:11] step 980/4768 loss 3.5061 lr 6.00e-04 gnorm 0.14 54,509 tok/s peakVRAM 13.7GB +[16:41:23] step 1000/4768 loss 3.4964 lr 6.00e-04 gnorm 0.13 54,509 tok/s peakVRAM 13.7GB +[16:44:36] step 1020/4768 loss 3.4905 lr 6.00e-04 gnorm 0.12 54,506 tok/s peakVRAM 13.7GB +[16:47:48] step 1040/4768 loss 3.4633 lr 6.00e-04 gnorm 0.12 54,505 tok/s peakVRAM 13.7GB +[16:51:01] step 1060/4768 loss 3.4752 lr 6.00e-04 gnorm 0.14 54,506 tok/s peakVRAM 13.7GB +[16:54:13] step 1080/4768 loss 3.4640 lr 6.00e-04 gnorm 0.13 54,502 tok/s peakVRAM 13.7GB +[16:57:25] step 1100/4768 loss 3.4485 lr 6.00e-04 gnorm 0.14 54,502 tok/s peakVRAM 13.7GB +[17:00:38] step 1120/4768 loss 3.4306 lr 6.00e-04 gnorm 0.11 54,497 tok/s peakVRAM 13.7GB +[17:03:50] step 1140/4768 loss 3.4239 lr 6.00e-04 gnorm 0.13 54,497 tok/s peakVRAM 13.7GB +[17:07:03] step 1160/4768 loss 3.4182 lr 6.00e-04 gnorm 0.12 54,502 tok/s peakVRAM 13.7GB +[17:10:15] step 1180/4768 loss 3.4205 lr 6.00e-04 gnorm 0.13 54,503 tok/s peakVRAM 13.7GB +[17:13:27] step 1200/4768 loss 3.4020 lr 6.00e-04 gnorm 0.12 54,501 tok/s peakVRAM 13.7GB +[17:16:40] step 1220/4768 loss 3.3999 lr 6.00e-04 gnorm 0.12 54,504 tok/s peakVRAM 13.7GB +[17:19:52] step 1240/4768 loss 3.3910 lr 6.00e-04 gnorm 0.13 54,502 tok/s peakVRAM 13.7GB +[17:23:05] step 1260/4768 loss 3.3769 lr 6.00e-04 gnorm 0.12 54,504 tok/s peakVRAM 13.7GB +[17:26:17] step 1280/4768 loss 3.3803 lr 6.00e-04 gnorm 0.11 54,504 tok/s peakVRAM 13.7GB +[17:29:29] step 1300/4768 loss 3.3732 lr 6.00e-04 gnorm 0.12 54,505 tok/s peakVRAM 13.7GB +[17:32:42] step 1320/4768 loss 3.3656 lr 6.00e-04 gnorm 0.11 54,502 tok/s peakVRAM 13.7GB +[17:35:54] step 1340/4768 loss 3.3657 lr 6.00e-04 gnorm 0.13 54,499 tok/s peakVRAM 13.7GB +[17:39:06] step 1360/4768 loss 3.3566 lr 6.00e-04 gnorm 0.11 54,498 tok/s peakVRAM 13.7GB +[17:42:19] step 1380/4768 loss 3.3479 lr 6.00e-04 gnorm 0.11 54,499 tok/s peakVRAM 13.7GB +[17:45:31] step 1400/4768 loss 3.3375 lr 6.00e-04 gnorm 0.12 54,505 tok/s peakVRAM 13.7GB +[17:48:44] step 1420/4768 loss 3.3260 lr 6.00e-04 gnorm 0.14 54,506 tok/s peakVRAM 13.7GB +[17:51:56] step 1440/4768 loss 3.3326 lr 6.00e-04 gnorm 0.10 54,506 tok/s peakVRAM 13.7GB +[17:55:08] step 1460/4768 loss 3.3299 lr 6.00e-04 gnorm 0.12 54,504 tok/s peakVRAM 13.7GB +[17:58:21] step 1480/4768 loss 3.3338 lr 6.00e-04 gnorm 0.11 54,505 tok/s peakVRAM 13.7GB +[18:01:33] step 1500/4768 loss 3.3074 lr 6.00e-04 gnorm 0.11 54,507 tok/s peakVRAM 13.7GB +[18:04:46] step 1520/4768 loss 3.3141 lr 6.00e-04 gnorm 0.11 54,506 tok/s peakVRAM 13.7GB +[18:07:58] step 1540/4768 loss 3.2981 lr 6.00e-04 gnorm 0.11 54,505 tok/s peakVRAM 13.7GB +[18:11:10] step 1560/4768 loss 3.3069 lr 6.00e-04 gnorm 0.11 54,502 tok/s peakVRAM 13.7GB +[18:14:23] step 1580/4768 loss 3.2950 lr 6.00e-04 gnorm 0.11 54,497 tok/s peakVRAM 13.7GB +[18:17:35] step 1600/4768 loss 3.2844 lr 6.00e-04 gnorm 0.11 54,499 tok/s peakVRAM 13.7GB +[18:20:48] step 1620/4768 loss 3.2848 lr 6.00e-04 gnorm 0.10 54,498 tok/s peakVRAM 13.7GB +[18:24:00] step 1640/4768 loss 3.2817 lr 6.00e-04 gnorm 0.10 54,501 tok/s peakVRAM 13.7GB +[18:27:12] step 1660/4768 loss 3.2826 lr 6.00e-04 gnorm 0.11 54,496 tok/s peakVRAM 13.7GB +[18:30:25] step 1680/4768 loss 3.2713 lr 6.00e-04 gnorm 0.12 54,497 tok/s peakVRAM 13.7GB +[18:33:37] step 1700/4768 loss 3.2644 lr 6.00e-04 gnorm 0.11 54,496 tok/s peakVRAM 13.7GB +[18:36:50] step 1720/4768 loss 3.2667 lr 6.00e-04 gnorm 0.12 54,494 tok/s peakVRAM 13.7GB +[18:40:02] step 1740/4768 loss 3.2596 lr 6.00e-04 gnorm 0.12 54,492 tok/s peakVRAM 13.7GB +[18:43:14] step 1760/4768 loss 3.2590 lr 6.00e-04 gnorm 0.10 54,497 tok/s peakVRAM 13.7GB +[18:46:27] step 1780/4768 loss 3.2461 lr 6.00e-04 gnorm 0.11 54,501 tok/s peakVRAM 13.7GB +[18:49:39] step 1800/4768 loss 3.2504 lr 6.00e-04 gnorm 0.11 54,498 tok/s peakVRAM 13.7GB +[18:52:52] step 1820/4768 loss 3.2456 lr 6.00e-04 gnorm 0.11 54,502 tok/s peakVRAM 13.7GB +[18:56:04] step 1840/4768 loss 3.2390 lr 6.00e-04 gnorm 0.11 54,501 tok/s peakVRAM 13.7GB +[18:59:16] step 1860/4768 loss 3.2379 lr 6.00e-04 gnorm 0.11 54,500 tok/s peakVRAM 13.7GB +[19:02:29] step 1880/4768 loss 3.2338 lr 6.00e-04 gnorm 0.11 54,502 tok/s peakVRAM 13.7GB +[19:05:41] step 1900/4768 loss 3.2283 lr 6.00e-04 gnorm 0.11 54,500 tok/s peakVRAM 13.7GB +[19:08:54] step 1920/4768 loss 3.2283 lr 6.00e-04 gnorm 0.10 54,502 tok/s peakVRAM 13.7GB +[19:12:06] step 1940/4768 loss 3.2208 lr 6.00e-04 gnorm 0.11 54,500 tok/s peakVRAM 13.7GB +[19:15:18] step 1960/4768 loss 3.2250 lr 6.00e-04 gnorm 0.11 54,500 tok/s peakVRAM 13.7GB +[19:18:31] step 1980/4768 loss 3.2196 lr 6.00e-04 gnorm 0.10 54,503 tok/s peakVRAM 13.7GB +[19:21:43] step 2000/4768 loss 3.2127 lr 6.00e-04 gnorm 0.11 54,502 tok/s peakVRAM 13.7GB +[19:24:56] step 2020/4768 loss 3.2123 lr 6.00e-04 gnorm 0.10 54,503 tok/s peakVRAM 13.7GB +[19:28:08] step 2040/4768 loss 3.2057 lr 6.00e-04 gnorm 0.11 54,505 tok/s peakVRAM 13.7GB +[19:31:20] step 2060/4768 loss 3.2061 lr 6.00e-04 gnorm 0.10 54,508 tok/s peakVRAM 13.7GB +[19:34:33] step 2080/4768 loss 3.1947 lr 6.00e-04 gnorm 0.10 54,507 tok/s peakVRAM 13.7GB +[19:37:45] step 2100/4768 loss 3.2031 lr 6.00e-04 gnorm 0.10 54,507 tok/s peakVRAM 13.7GB +[19:40:57] step 2120/4768 loss 3.1923 lr 6.00e-04 gnorm 0.11 54,510 tok/s peakVRAM 13.7GB +[19:44:10] step 2140/4768 loss 3.2055 lr 6.00e-04 gnorm 0.11 54,509 tok/s peakVRAM 13.7GB +[19:47:22] step 2160/4768 loss 3.1841 lr 6.00e-04 gnorm 0.11 54,506 tok/s peakVRAM 13.7GB +[19:50:35] step 2180/4768 loss 3.1859 lr 6.00e-04 gnorm 0.11 54,503 tok/s peakVRAM 13.7GB +[19:53:47] step 2200/4768 loss 3.1875 lr 6.00e-04 gnorm 0.10 54,505 tok/s peakVRAM 13.7GB +[19:56:59] step 2220/4768 loss 3.1795 lr 6.00e-04 gnorm 0.11 54,505 tok/s peakVRAM 13.7GB +[20:00:12] step 2240/4768 loss 3.1762 lr 6.00e-04 gnorm 0.10 54,506 tok/s peakVRAM 13.7GB +[20:03:24] step 2260/4768 loss 3.1724 lr 6.00e-04 gnorm 0.10 54,505 tok/s peakVRAM 13.7GB +[20:06:36] step 2280/4768 loss 3.1688 lr 6.00e-04 gnorm 0.09 54,507 tok/s peakVRAM 13.7GB +[20:09:49] step 2300/4768 loss 3.1722 lr 6.00e-04 gnorm 0.11 54,505 tok/s peakVRAM 13.7GB +[20:13:01] step 2320/4768 loss 3.1690 lr 6.00e-04 gnorm 0.11 54,506 tok/s peakVRAM 13.7GB +[20:16:14] step 2340/4768 loss 3.1635 lr 6.00e-04 gnorm 0.10 54,504 tok/s peakVRAM 13.7GB +[20:19:26] step 2360/4768 loss 3.1542 lr 6.00e-04 gnorm 0.10 54,504 tok/s peakVRAM 13.7GB +[20:22:38] step 2380/4768 loss 3.1533 lr 6.00e-04 gnorm 0.09 54,501 tok/s peakVRAM 13.7GB +[20:25:51] step 2400/4768 loss 3.1534 lr 6.00e-04 gnorm 0.09 54,503 tok/s peakVRAM 13.7GB +[20:29:03] step 2420/4768 loss 3.1563 lr 6.00e-04 gnorm 0.09 54,498 tok/s peakVRAM 13.7GB +[20:32:16] step 2440/4768 loss 3.1502 lr 6.00e-04 gnorm 0.10 54,500 tok/s peakVRAM 13.7GB +[20:35:28] step 2460/4768 loss 3.1489 lr 6.00e-04 gnorm 0.10 54,499 tok/s peakVRAM 13.7GB +[20:38:40] step 2480/4768 loss 3.1461 lr 6.00e-04 gnorm 0.10 54,500 tok/s peakVRAM 13.7GB +[20:41:53] step 2500/4768 loss 3.1406 lr 6.00e-04 gnorm 0.10 54,500 tok/s peakVRAM 13.7GB +[20:45:05] step 2520/4768 loss 3.1425 lr 6.00e-04 gnorm 0.10 54,504 tok/s peakVRAM 13.7GB +[20:48:18] step 2540/4768 loss 3.1422 lr 6.00e-04 gnorm 0.09 54,503 tok/s peakVRAM 13.7GB +[20:51:30] step 2560/4768 loss 3.1397 lr 6.00e-04 gnorm 0.10 54,504 tok/s peakVRAM 13.7GB +[20:54:42] step 2580/4768 loss 3.1356 lr 6.00e-04 gnorm 0.10 54,502 tok/s peakVRAM 13.7GB +[20:57:55] step 2600/4768 loss 3.1255 lr 6.00e-04 gnorm 0.10 54,505 tok/s peakVRAM 13.7GB +[21:01:07] step 2620/4768 loss 3.1343 lr 6.00e-04 gnorm 0.10 54,503 tok/s peakVRAM 13.7GB +[21:04:20] step 2640/4768 loss 3.1348 lr 6.00e-04 gnorm 0.10 54,502 tok/s peakVRAM 13.7GB +[21:07:32] step 2660/4768 loss 3.1334 lr 6.00e-04 gnorm 0.11 54,499 tok/s peakVRAM 13.7GB +[21:10:44] step 2680/4768 loss 3.1281 lr 6.00e-04 gnorm 0.10 54,502 tok/s peakVRAM 13.7GB +[21:13:57] step 2700/4768 loss 3.1312 lr 6.00e-04 gnorm 0.10 54,503 tok/s peakVRAM 13.7GB +[21:17:09] step 2720/4768 loss 3.1138 lr 6.00e-04 gnorm 0.09 54,499 tok/s peakVRAM 13.7GB +[21:20:22] step 2740/4768 loss 3.1155 lr 6.00e-04 gnorm 0.10 54,502 tok/s peakVRAM 13.7GB +[21:23:34] step 2760/4768 loss 3.1185 lr 6.00e-04 gnorm 0.09 54,500 tok/s peakVRAM 13.7GB +[21:26:46] step 2780/4768 loss 3.1098 lr 6.00e-04 gnorm 0.11 54,502 tok/s peakVRAM 13.7GB +[21:29:59] step 2800/4768 loss 3.1045 lr 6.00e-04 gnorm 0.11 54,500 tok/s peakVRAM 13.7GB +[21:33:11] step 2820/4768 loss 3.1017 lr 6.00e-04 gnorm 0.09 54,501 tok/s peakVRAM 13.7GB +[21:36:23] step 2840/4768 loss 3.1099 lr 6.00e-04 gnorm 0.10 54,503 tok/s peakVRAM 13.7GB +[21:39:36] step 2860/4768 loss 3.1015 lr 6.00e-04 gnorm 0.10 54,503 tok/s peakVRAM 13.7GB +[21:42:48] step 2880/4768 loss 3.1037 lr 6.00e-04 gnorm 0.10 54,501 tok/s peakVRAM 13.7GB +[21:46:01] step 2900/4768 loss 3.1048 lr 6.00e-04 gnorm 0.09 54,497 tok/s peakVRAM 13.7GB +[21:49:13] step 2920/4768 loss 3.1120 lr 6.00e-04 gnorm 0.10 54,500 tok/s peakVRAM 13.7GB +[21:52:25] step 2940/4768 loss 3.1023 lr 6.00e-04 gnorm 0.09 54,505 tok/s peakVRAM 13.7GB +[21:55:38] step 2960/4768 loss 3.1040 lr 6.00e-04 gnorm 0.10 54,500 tok/s peakVRAM 13.7GB +[21:58:50] step 2980/4768 loss 3.0913 lr 6.00e-04 gnorm 0.10 54,500 tok/s peakVRAM 13.7GB +[22:02:03] step 3000/4768 loss 3.0920 lr 6.00e-04 gnorm 0.09 54,502 tok/s peakVRAM 13.7GB +[22:05:15] step 3020/4768 loss 3.0874 lr 6.00e-04 gnorm 0.11 54,504 tok/s peakVRAM 13.7GB +[22:08:27] step 3040/4768 loss 3.0845 lr 6.00e-04 gnorm 0.09 54,501 tok/s peakVRAM 13.7GB +[22:11:40] step 3060/4768 loss 3.0876 lr 6.00e-04 gnorm 0.09 54,503 tok/s peakVRAM 13.7GB +[22:14:52] step 3080/4768 loss 3.0810 lr 6.00e-04 gnorm 0.09 54,507 tok/s peakVRAM 13.7GB +[22:18:05] step 3100/4768 loss 3.0741 lr 6.00e-04 gnorm 0.11 54,507 tok/s peakVRAM 13.7GB +[22:21:17] step 3120/4768 loss 3.0905 lr 6.00e-04 gnorm 0.10 54,507 tok/s peakVRAM 13.7GB +[22:24:29] step 3140/4768 loss 3.0764 lr 6.00e-04 gnorm 0.09 54,506 tok/s peakVRAM 13.7GB +[22:27:42] step 3160/4768 loss 3.0722 lr 6.00e-04 gnorm 0.10 54,504 tok/s peakVRAM 13.7GB +[22:30:54] step 3180/4768 loss 3.0783 lr 6.00e-04 gnorm 0.10 54,505 tok/s peakVRAM 13.7GB +[22:34:06] step 3200/4768 loss 3.0794 lr 6.00e-04 gnorm 0.09 54,506 tok/s peakVRAM 13.7GB +[22:37:19] step 3220/4768 loss 3.0889 lr 6.00e-04 gnorm 0.09 54,507 tok/s peakVRAM 13.7GB +[22:40:31] step 3240/4768 loss 3.0708 lr 6.00e-04 gnorm 0.09 54,504 tok/s peakVRAM 13.7GB +[22:43:44] step 3260/4768 loss 3.0677 lr 6.00e-04 gnorm 0.09 54,503 tok/s peakVRAM 13.7GB +[22:46:56] step 3280/4768 loss 3.0627 lr 6.00e-04 gnorm 0.09 54,504 tok/s peakVRAM 13.7GB +[22:50:08] step 3300/4768 loss 3.0740 lr 6.00e-04 gnorm 0.09 54,506 tok/s peakVRAM 13.7GB +[22:53:21] step 3320/4768 loss 3.0677 lr 6.00e-04 gnorm 0.09 54,505 tok/s peakVRAM 13.7GB +[22:56:33] step 3340/4768 loss 3.0681 lr 6.00e-04 gnorm 0.09 54,504 tok/s peakVRAM 13.7GB +[22:59:46] step 3360/4768 loss 3.0593 lr 6.00e-04 gnorm 0.10 54,507 tok/s peakVRAM 13.7GB +[23:02:58] step 3380/4768 loss 3.0687 lr 6.00e-04 gnorm 0.10 54,505 tok/s peakVRAM 13.7GB +[23:06:10] step 3400/4768 loss 3.0538 lr 6.00e-04 gnorm 0.09 54,504 tok/s peakVRAM 13.7GB +[23:09:23] step 3420/4768 loss 3.0617 lr 6.00e-04 gnorm 0.09 54,504 tok/s peakVRAM 13.7GB +[23:12:35] step 3440/4768 loss 3.0533 lr 6.00e-04 gnorm 0.10 54,503 tok/s peakVRAM 13.7GB +[23:15:47] step 3460/4768 loss 3.0629 lr 6.00e-04 gnorm 0.09 54,506 tok/s peakVRAM 13.7GB +[23:19:00] step 3480/4768 loss 3.0618 lr 6.00e-04 gnorm 0.08 54,506 tok/s peakVRAM 13.7GB +[23:22:12] step 3500/4768 loss 3.0479 lr 6.00e-04 gnorm 0.09 54,505 tok/s peakVRAM 13.7GB +[23:25:25] step 3520/4768 loss 3.0510 lr 6.00e-04 gnorm 0.09 54,505 tok/s peakVRAM 13.7GB +[23:28:37] step 3540/4768 loss 3.0538 lr 6.00e-04 gnorm 0.10 54,505 tok/s peakVRAM 13.7GB +[23:31:49] step 3560/4768 loss 3.0581 lr 6.00e-04 gnorm 0.09 54,503 tok/s peakVRAM 13.7GB +[23:35:02] step 3580/4768 loss 3.0513 lr 6.00e-04 gnorm 0.08 54,502 tok/s peakVRAM 13.7GB +[23:38:14] step 3600/4768 loss 3.0537 lr 6.00e-04 gnorm 0.08 54,506 tok/s peakVRAM 13.7GB +[23:41:27] step 3620/4768 loss 3.0451 lr 6.00e-04 gnorm 0.09 54,506 tok/s peakVRAM 13.7GB +[23:44:39] step 3640/4768 loss 3.0531 lr 6.00e-04 gnorm 0.09 54,506 tok/s peakVRAM 13.7GB +[23:47:51] step 3660/4768 loss 3.0418 lr 6.00e-04 gnorm 0.09 54,508 tok/s peakVRAM 13.7GB +[23:51:04] step 3680/4768 loss 3.0485 lr 6.00e-04 gnorm 0.09 54,510 tok/s peakVRAM 13.7GB +[23:54:16] step 3700/4768 loss 3.0457 lr 6.00e-04 gnorm 0.09 54,511 tok/s peakVRAM 13.7GB +[23:57:28] step 3720/4768 loss 3.0391 lr 6.00e-04 gnorm 0.09 54,509 tok/s peakVRAM 13.7GB +[00:00:41] step 3740/4768 loss 3.0351 lr 6.00e-04 gnorm 0.09 54,507 tok/s peakVRAM 13.7GB +[00:03:53] step 3760/4768 loss 3.0340 lr 6.00e-04 gnorm 0.09 54,508 tok/s peakVRAM 13.7GB +[00:07:05] step 3780/4768 loss 3.0346 lr 6.00e-04 gnorm 0.09 54,508 tok/s peakVRAM 13.7GB +[00:10:18] step 3800/4768 loss 3.0353 lr 6.00e-04 gnorm 0.08 54,509 tok/s peakVRAM 13.7GB +[00:13:30] step 3820/4768 loss 3.0289 lr 5.61e-04 gnorm 0.09 54,503 tok/s peakVRAM 13.7GB +[00:16:43] step 3840/4768 loss 3.0271 lr 5.13e-04 gnorm 0.08 54,505 tok/s peakVRAM 13.7GB +[00:19:55] step 3860/4768 loss 3.0204 lr 4.83e-04 gnorm 0.07 54,505 tok/s peakVRAM 13.7GB +[00:23:07] step 3880/4768 loss 3.0173 lr 4.59e-04 gnorm 0.07 54,507 tok/s peakVRAM 13.7GB +[00:26:20] step 3900/4768 loss 3.0108 lr 4.39e-04 gnorm 0.08 54,502 tok/s peakVRAM 13.7GB +[00:29:32] step 3920/4768 loss 3.0052 lr 4.21e-04 gnorm 0.08 54,504 tok/s peakVRAM 13.7GB +[00:32:45] step 3940/4768 loss 2.9974 lr 4.05e-04 gnorm 0.07 54,507 tok/s peakVRAM 13.7GB +[00:35:57] step 3960/4768 loss 2.9981 lr 3.89e-04 gnorm 0.08 54,506 tok/s peakVRAM 13.7GB +[00:39:09] step 3980/4768 loss 2.9899 lr 3.75e-04 gnorm 0.08 54,503 tok/s peakVRAM 13.7GB +[00:42:22] step 4000/4768 loss 2.9894 lr 3.62e-04 gnorm 0.07 54,504 tok/s peakVRAM 13.7GB +[00:45:34] step 4020/4768 loss 2.9862 lr 3.50e-04 gnorm 0.07 54,505 tok/s peakVRAM 13.7GB +[00:48:46] step 4040/4768 loss 2.9767 lr 3.38e-04 gnorm 0.07 54,504 tok/s peakVRAM 13.7GB +[00:51:59] step 4060/4768 loss 2.9768 lr 3.26e-04 gnorm 0.07 54,504 tok/s peakVRAM 13.7GB +[00:55:11] step 4080/4768 loss 2.9678 lr 3.15e-04 gnorm 0.07 54,506 tok/s peakVRAM 13.7GB +[00:58:24] step 4100/4768 loss 2.9733 lr 3.05e-04 gnorm 0.07 54,503 tok/s peakVRAM 13.7GB +[01:01:36] step 4120/4768 loss 2.9621 lr 2.95e-04 gnorm 0.07 54,504 tok/s peakVRAM 13.7GB +[01:04:48] step 4140/4768 loss 2.9543 lr 2.85e-04 gnorm 0.07 54,505 tok/s peakVRAM 13.7GB +[01:08:01] step 4160/4768 loss 2.9723 lr 2.75e-04 gnorm 0.07 54,506 tok/s peakVRAM 13.7GB +[01:11:13] step 4180/4768 loss 2.9649 lr 2.66e-04 gnorm 0.07 54,508 tok/s peakVRAM 13.7GB +[01:14:25] step 4200/4768 loss 2.9585 lr 2.57e-04 gnorm 0.07 54,509 tok/s peakVRAM 13.7GB +[01:17:38] step 4220/4768 loss 2.9518 lr 2.48e-04 gnorm 0.07 54,510 tok/s peakVRAM 13.7GB +[01:20:50] step 4240/4768 loss 2.9587 lr 2.40e-04 gnorm 0.07 54,507 tok/s peakVRAM 13.7GB +[01:24:03] step 4260/4768 loss 2.9504 lr 2.31e-04 gnorm 0.07 54,510 tok/s peakVRAM 13.7GB +[01:27:15] step 4280/4768 loss 2.9496 lr 2.23e-04 gnorm 0.07 54,512 tok/s peakVRAM 13.7GB +[01:30:27] step 4300/4768 loss 2.9478 lr 2.15e-04 gnorm 0.07 54,510 tok/s peakVRAM 13.7GB +[01:33:40] step 4320/4768 loss 2.9426 lr 2.07e-04 gnorm 0.07 54,509 tok/s peakVRAM 13.7GB +[01:36:52] step 4340/4768 loss 2.9412 lr 1.99e-04 gnorm 0.07 54,507 tok/s peakVRAM 13.7GB +[01:40:04] step 4360/4768 loss 2.9350 lr 1.92e-04 gnorm 0.07 54,505 tok/s peakVRAM 13.7GB +[01:43:17] step 4380/4768 loss 2.9421 lr 1.84e-04 gnorm 0.07 54,509 tok/s peakVRAM 13.7GB +[01:46:29] step 4400/4768 loss 2.9398 lr 1.77e-04 gnorm 0.07 54,509 tok/s peakVRAM 13.7GB +[01:49:42] step 4420/4768 loss 2.9329 lr 1.70e-04 gnorm 0.07 54,506 tok/s peakVRAM 13.7GB +[01:52:54] step 4440/4768 loss 2.9391 lr 1.63e-04 gnorm 0.07 54,507 tok/s peakVRAM 13.7GB +[01:56:06] step 4460/4768 loss 2.9348 lr 1.56e-04 gnorm 0.07 54,505 tok/s peakVRAM 13.7GB +[01:59:19] step 4480/4768 loss 2.9299 lr 1.49e-04 gnorm 0.07 54,504 tok/s peakVRAM 13.7GB +[02:02:31] step 4500/4768 loss 2.9271 lr 1.42e-04 gnorm 0.07 54,502 tok/s peakVRAM 13.7GB +[02:05:43] step 4520/4768 loss 2.9291 lr 1.36e-04 gnorm 0.07 54,504 tok/s peakVRAM 13.7GB +[02:08:56] step 4540/4768 loss 2.9205 lr 1.29e-04 gnorm 0.07 54,503 tok/s peakVRAM 13.7GB +[02:12:08] step 4560/4768 loss 2.9235 lr 1.23e-04 gnorm 0.07 54,504 tok/s peakVRAM 13.7GB +[02:15:21] step 4580/4768 loss 2.9269 lr 1.16e-04 gnorm 0.07 54,504 tok/s peakVRAM 13.7GB +[02:18:33] step 4600/4768 loss 2.9252 lr 1.10e-04 gnorm 0.07 54,506 tok/s peakVRAM 13.7GB +[02:21:45] step 4620/4768 loss 2.9120 lr 1.04e-04 gnorm 0.07 54,507 tok/s peakVRAM 13.7GB +[02:24:58] step 4640/4768 loss 2.9213 lr 9.78e-05 gnorm 0.07 54,508 tok/s peakVRAM 13.7GB +[02:28:10] step 4660/4768 loss 2.9140 lr 9.18e-05 gnorm 0.07 54,508 tok/s peakVRAM 13.7GB +[02:31:22] step 4680/4768 loss 2.9119 lr 8.58e-05 gnorm 0.07 54,508 tok/s peakVRAM 13.7GB +[02:34:35] step 4700/4768 loss 2.9078 lr 7.99e-05 gnorm 0.07 54,507 tok/s peakVRAM 13.7GB +[02:37:47] step 4720/4768 loss 2.9089 lr 7.41e-05 gnorm 0.07 54,505 tok/s peakVRAM 13.7GB +[02:41:00] step 4740/4768 loss 2.9066 lr 6.83e-05 gnorm 0.07 54,507 tok/s peakVRAM 13.7GB +[02:44:12] step 4760/4768 loss 2.9157 lr 6.26e-05 gnorm 0.07 54,512 tok/s peakVRAM 13.7GB +[02:45:31] ckpt @ 4768 +[02:45:31] DONE diff --git a/R1/run/train.out b/R1/run/train.out new file mode 100644 index 0000000000000000000000000000000000000000..684af70519c6b1198dd6d413eb6bf8456b6dda58 --- /dev/null +++ b/R1/run/train.out @@ -0,0 +1,248 @@ +[13:59:56] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[13:59:56] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[13:59:58] model GPT-ref: 183.6M param (L16 d896 h14) +[13:59:59] optimizer=muon muon_lr=0.02 (mult=33.3x) +[13:59:59] r11 pamiec: act_ckpt=True loss_chunk=1024 +[13:59:59] r13: mixer_layout=AAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[13:59:59] torch.compile enabled +[13:59:59] START gpt-ref: steps=4768 (od 0) lr=0.0006->6e-05 +[14:04:20] step 20/4768 loss 10.3757 lr 2.40e-05 gnorm 2.72 40,193 tok/s peakVRAM 13.7GB +[14:07:32] step 40/4768 loss 9.5564 lr 4.80e-05 gnorm 1.29 54,573 tok/s peakVRAM 13.7GB +[14:10:45] step 60/4768 loss 8.6944 lr 7.20e-05 gnorm 0.71 54,577 tok/s peakVRAM 13.7GB +[14:13:57] step 80/4768 loss 8.0439 lr 9.60e-05 gnorm 1.18 54,575 tok/s peakVRAM 13.7GB +[14:17:09] step 100/4768 loss 7.5900 lr 1.20e-04 gnorm 0.92 54,564 tok/s peakVRAM 13.7GB +[14:20:21] step 120/4768 loss 7.1203 lr 1.44e-04 gnorm 0.82 54,555 tok/s peakVRAM 13.7GB +[14:23:33] step 140/4768 loss 6.7051 lr 1.68e-04 gnorm 0.74 54,546 tok/s peakVRAM 13.7GB +[14:26:46] step 160/4768 loss 6.3498 lr 1.92e-04 gnorm 0.75 54,544 tok/s peakVRAM 13.7GB +[14:29:58] step 180/4768 loss 6.0849 lr 2.16e-04 gnorm 0.78 54,537 tok/s peakVRAM 13.7GB +[14:33:10] step 200/4768 loss 5.8639 lr 2.40e-04 gnorm 0.87 54,535 tok/s peakVRAM 13.7GB +[14:36:22] step 220/4768 loss 5.6756 lr 2.64e-04 gnorm 0.87 54,530 tok/s peakVRAM 13.7GB +[14:39:35] step 240/4768 loss 5.4997 lr 2.88e-04 gnorm 0.77 54,530 tok/s peakVRAM 13.7GB +[14:42:47] step 260/4768 loss 5.3285 lr 3.12e-04 gnorm 0.84 54,528 tok/s peakVRAM 13.7GB +[14:45:59] step 280/4768 loss 5.1732 lr 3.36e-04 gnorm 0.91 54,528 tok/s peakVRAM 13.7GB +[14:49:12] step 300/4768 loss 5.0283 lr 3.60e-04 gnorm 0.78 54,530 tok/s peakVRAM 13.7GB +[14:52:24] step 320/4768 loss 4.8935 lr 3.84e-04 gnorm 0.63 54,530 tok/s peakVRAM 13.7GB +[14:55:36] step 340/4768 loss 4.7505 lr 4.08e-04 gnorm 0.59 54,527 tok/s peakVRAM 13.7GB +[14:58:49] step 360/4768 loss 4.6245 lr 4.32e-04 gnorm 0.72 54,527 tok/s peakVRAM 13.7GB +[15:02:01] step 380/4768 loss 4.5067 lr 4.56e-04 gnorm 0.46 54,527 tok/s peakVRAM 13.7GB +[15:05:13] step 400/4768 loss 4.4020 lr 4.80e-04 gnorm 0.52 54,528 tok/s peakVRAM 13.7GB +[15:08:25] step 420/4768 loss 4.3259 lr 5.04e-04 gnorm 0.47 54,524 tok/s peakVRAM 13.7GB +[15:11:38] step 440/4768 loss 4.2479 lr 5.28e-04 gnorm 0.40 54,526 tok/s peakVRAM 13.7GB +[15:14:50] step 460/4768 loss 4.1796 lr 5.52e-04 gnorm 0.41 54,527 tok/s peakVRAM 13.7GB +[15:18:02] step 480/4768 loss 4.1299 lr 5.76e-04 gnorm 0.42 54,525 tok/s peakVRAM 13.7GB +[15:21:15] step 500/4768 loss 4.0777 lr 6.00e-04 gnorm 0.36 54,523 tok/s peakVRAM 13.7GB +[15:24:27] step 520/4768 loss 4.0391 lr 6.00e-04 gnorm 0.31 54,520 tok/s peakVRAM 13.7GB +[15:27:39] step 540/4768 loss 3.9885 lr 6.00e-04 gnorm 0.26 54,520 tok/s peakVRAM 13.7GB +[15:30:52] step 560/4768 loss 3.9369 lr 6.00e-04 gnorm 0.20 54,521 tok/s peakVRAM 13.7GB +[15:34:04] step 580/4768 loss 3.8963 lr 6.00e-04 gnorm 0.22 54,519 tok/s peakVRAM 13.7GB +[15:37:16] step 600/4768 loss 3.8546 lr 6.00e-04 gnorm 0.23 54,522 tok/s peakVRAM 13.7GB +[15:40:29] step 620/4768 loss 3.8328 lr 6.00e-04 gnorm 0.21 54,520 tok/s peakVRAM 13.7GB +[15:43:41] step 640/4768 loss 3.8063 lr 6.00e-04 gnorm 0.19 54,519 tok/s peakVRAM 13.7GB +[15:46:53] step 660/4768 loss 3.7745 lr 6.00e-04 gnorm 0.20 54,520 tok/s peakVRAM 13.7GB +[15:50:06] step 680/4768 loss 3.7530 lr 6.00e-04 gnorm 0.17 54,519 tok/s peakVRAM 13.7GB +[15:53:18] step 700/4768 loss 3.7221 lr 6.00e-04 gnorm 0.19 54,519 tok/s peakVRAM 13.7GB +[15:56:30] step 720/4768 loss 3.6912 lr 6.00e-04 gnorm 0.16 54,517 tok/s peakVRAM 13.7GB +[15:59:43] step 740/4768 loss 3.6873 lr 6.00e-04 gnorm 0.18 54,513 tok/s peakVRAM 13.7GB +[16:02:55] step 760/4768 loss 3.6627 lr 6.00e-04 gnorm 0.17 54,513 tok/s peakVRAM 13.7GB +[16:06:07] step 780/4768 loss 3.6369 lr 6.00e-04 gnorm 0.15 54,512 tok/s peakVRAM 13.7GB +[16:09:20] step 800/4768 loss 3.6232 lr 6.00e-04 gnorm 0.16 54,508 tok/s peakVRAM 13.7GB +[16:12:32] step 820/4768 loss 3.6076 lr 6.00e-04 gnorm 0.15 54,512 tok/s peakVRAM 13.7GB +[16:15:45] step 840/4768 loss 3.5929 lr 6.00e-04 gnorm 0.15 54,510 tok/s peakVRAM 13.7GB +[16:18:57] step 860/4768 loss 3.5715 lr 6.00e-04 gnorm 0.13 54,510 tok/s peakVRAM 13.7GB +[16:22:09] step 880/4768 loss 3.5661 lr 6.00e-04 gnorm 0.15 54,508 tok/s peakVRAM 13.7GB +[16:25:22] step 900/4768 loss 3.5479 lr 6.00e-04 gnorm 0.14 54,510 tok/s peakVRAM 13.7GB +[16:28:34] step 920/4768 loss 3.5404 lr 6.00e-04 gnorm 0.13 54,511 tok/s peakVRAM 13.7GB +[16:31:46] step 940/4768 loss 3.5349 lr 6.00e-04 gnorm 0.13 54,511 tok/s peakVRAM 13.7GB +[16:34:59] step 960/4768 loss 3.5101 lr 6.00e-04 gnorm 0.14 54,511 tok/s peakVRAM 13.7GB +[16:38:11] step 980/4768 loss 3.5061 lr 6.00e-04 gnorm 0.14 54,509 tok/s peakVRAM 13.7GB +[16:41:23] step 1000/4768 loss 3.4964 lr 6.00e-04 gnorm 0.13 54,509 tok/s peakVRAM 13.7GB +[16:44:36] step 1020/4768 loss 3.4905 lr 6.00e-04 gnorm 0.12 54,506 tok/s peakVRAM 13.7GB +[16:47:48] step 1040/4768 loss 3.4633 lr 6.00e-04 gnorm 0.12 54,505 tok/s peakVRAM 13.7GB +[16:51:01] step 1060/4768 loss 3.4752 lr 6.00e-04 gnorm 0.14 54,506 tok/s peakVRAM 13.7GB +[16:54:13] step 1080/4768 loss 3.4640 lr 6.00e-04 gnorm 0.13 54,502 tok/s peakVRAM 13.7GB +[16:57:25] step 1100/4768 loss 3.4485 lr 6.00e-04 gnorm 0.14 54,502 tok/s peakVRAM 13.7GB +[17:00:38] step 1120/4768 loss 3.4306 lr 6.00e-04 gnorm 0.11 54,497 tok/s peakVRAM 13.7GB +[17:03:50] step 1140/4768 loss 3.4239 lr 6.00e-04 gnorm 0.13 54,497 tok/s peakVRAM 13.7GB +[17:07:03] step 1160/4768 loss 3.4182 lr 6.00e-04 gnorm 0.12 54,502 tok/s peakVRAM 13.7GB +[17:10:15] step 1180/4768 loss 3.4205 lr 6.00e-04 gnorm 0.13 54,503 tok/s peakVRAM 13.7GB +[17:13:27] step 1200/4768 loss 3.4020 lr 6.00e-04 gnorm 0.12 54,501 tok/s peakVRAM 13.7GB +[17:16:40] step 1220/4768 loss 3.3999 lr 6.00e-04 gnorm 0.12 54,504 tok/s peakVRAM 13.7GB +[17:19:52] step 1240/4768 loss 3.3910 lr 6.00e-04 gnorm 0.13 54,502 tok/s peakVRAM 13.7GB +[17:23:05] step 1260/4768 loss 3.3769 lr 6.00e-04 gnorm 0.12 54,504 tok/s peakVRAM 13.7GB +[17:26:17] step 1280/4768 loss 3.3803 lr 6.00e-04 gnorm 0.11 54,504 tok/s peakVRAM 13.7GB +[17:29:29] step 1300/4768 loss 3.3732 lr 6.00e-04 gnorm 0.12 54,505 tok/s peakVRAM 13.7GB +[17:32:42] step 1320/4768 loss 3.3656 lr 6.00e-04 gnorm 0.11 54,502 tok/s peakVRAM 13.7GB +[17:35:54] step 1340/4768 loss 3.3657 lr 6.00e-04 gnorm 0.13 54,499 tok/s peakVRAM 13.7GB +[17:39:06] step 1360/4768 loss 3.3566 lr 6.00e-04 gnorm 0.11 54,498 tok/s peakVRAM 13.7GB +[17:42:19] step 1380/4768 loss 3.3479 lr 6.00e-04 gnorm 0.11 54,499 tok/s peakVRAM 13.7GB +[17:45:31] step 1400/4768 loss 3.3375 lr 6.00e-04 gnorm 0.12 54,505 tok/s peakVRAM 13.7GB +[17:48:44] step 1420/4768 loss 3.3260 lr 6.00e-04 gnorm 0.14 54,506 tok/s peakVRAM 13.7GB +[17:51:56] step 1440/4768 loss 3.3326 lr 6.00e-04 gnorm 0.10 54,506 tok/s peakVRAM 13.7GB +[17:55:08] step 1460/4768 loss 3.3299 lr 6.00e-04 gnorm 0.12 54,504 tok/s peakVRAM 13.7GB +[17:58:21] step 1480/4768 loss 3.3338 lr 6.00e-04 gnorm 0.11 54,505 tok/s peakVRAM 13.7GB +[18:01:33] step 1500/4768 loss 3.3074 lr 6.00e-04 gnorm 0.11 54,507 tok/s peakVRAM 13.7GB +[18:04:46] step 1520/4768 loss 3.3141 lr 6.00e-04 gnorm 0.11 54,506 tok/s peakVRAM 13.7GB +[18:07:58] step 1540/4768 loss 3.2981 lr 6.00e-04 gnorm 0.11 54,505 tok/s peakVRAM 13.7GB +[18:11:10] step 1560/4768 loss 3.3069 lr 6.00e-04 gnorm 0.11 54,502 tok/s peakVRAM 13.7GB +[18:14:23] step 1580/4768 loss 3.2950 lr 6.00e-04 gnorm 0.11 54,497 tok/s peakVRAM 13.7GB +[18:17:35] step 1600/4768 loss 3.2844 lr 6.00e-04 gnorm 0.11 54,499 tok/s peakVRAM 13.7GB +[18:20:48] step 1620/4768 loss 3.2848 lr 6.00e-04 gnorm 0.10 54,498 tok/s peakVRAM 13.7GB +[18:24:00] step 1640/4768 loss 3.2817 lr 6.00e-04 gnorm 0.10 54,501 tok/s peakVRAM 13.7GB +[18:27:12] step 1660/4768 loss 3.2826 lr 6.00e-04 gnorm 0.11 54,496 tok/s peakVRAM 13.7GB +[18:30:25] step 1680/4768 loss 3.2713 lr 6.00e-04 gnorm 0.12 54,497 tok/s peakVRAM 13.7GB +[18:33:37] step 1700/4768 loss 3.2644 lr 6.00e-04 gnorm 0.11 54,496 tok/s peakVRAM 13.7GB +[18:36:50] step 1720/4768 loss 3.2667 lr 6.00e-04 gnorm 0.12 54,494 tok/s peakVRAM 13.7GB +[18:40:02] step 1740/4768 loss 3.2596 lr 6.00e-04 gnorm 0.12 54,492 tok/s peakVRAM 13.7GB +[18:43:14] step 1760/4768 loss 3.2590 lr 6.00e-04 gnorm 0.10 54,497 tok/s peakVRAM 13.7GB +[18:46:27] step 1780/4768 loss 3.2461 lr 6.00e-04 gnorm 0.11 54,501 tok/s peakVRAM 13.7GB +[18:49:39] step 1800/4768 loss 3.2504 lr 6.00e-04 gnorm 0.11 54,498 tok/s peakVRAM 13.7GB +[18:52:52] step 1820/4768 loss 3.2456 lr 6.00e-04 gnorm 0.11 54,502 tok/s peakVRAM 13.7GB +[18:56:04] step 1840/4768 loss 3.2390 lr 6.00e-04 gnorm 0.11 54,501 tok/s peakVRAM 13.7GB +[18:59:16] step 1860/4768 loss 3.2379 lr 6.00e-04 gnorm 0.11 54,500 tok/s peakVRAM 13.7GB +[19:02:29] step 1880/4768 loss 3.2338 lr 6.00e-04 gnorm 0.11 54,502 tok/s peakVRAM 13.7GB +[19:05:41] step 1900/4768 loss 3.2283 lr 6.00e-04 gnorm 0.11 54,500 tok/s peakVRAM 13.7GB +[19:08:54] step 1920/4768 loss 3.2283 lr 6.00e-04 gnorm 0.10 54,502 tok/s peakVRAM 13.7GB +[19:12:06] step 1940/4768 loss 3.2208 lr 6.00e-04 gnorm 0.11 54,500 tok/s peakVRAM 13.7GB +[19:15:18] step 1960/4768 loss 3.2250 lr 6.00e-04 gnorm 0.11 54,500 tok/s peakVRAM 13.7GB +[19:18:31] step 1980/4768 loss 3.2196 lr 6.00e-04 gnorm 0.10 54,503 tok/s peakVRAM 13.7GB +[19:21:43] step 2000/4768 loss 3.2127 lr 6.00e-04 gnorm 0.11 54,502 tok/s peakVRAM 13.7GB +[19:24:56] step 2020/4768 loss 3.2123 lr 6.00e-04 gnorm 0.10 54,503 tok/s peakVRAM 13.7GB +[19:28:08] step 2040/4768 loss 3.2057 lr 6.00e-04 gnorm 0.11 54,505 tok/s peakVRAM 13.7GB +[19:31:20] step 2060/4768 loss 3.2061 lr 6.00e-04 gnorm 0.10 54,508 tok/s peakVRAM 13.7GB +[19:34:33] step 2080/4768 loss 3.1947 lr 6.00e-04 gnorm 0.10 54,507 tok/s peakVRAM 13.7GB +[19:37:45] step 2100/4768 loss 3.2031 lr 6.00e-04 gnorm 0.10 54,507 tok/s peakVRAM 13.7GB +[19:40:57] step 2120/4768 loss 3.1923 lr 6.00e-04 gnorm 0.11 54,510 tok/s peakVRAM 13.7GB +[19:44:10] step 2140/4768 loss 3.2055 lr 6.00e-04 gnorm 0.11 54,509 tok/s peakVRAM 13.7GB +[19:47:22] step 2160/4768 loss 3.1841 lr 6.00e-04 gnorm 0.11 54,506 tok/s peakVRAM 13.7GB +[19:50:35] step 2180/4768 loss 3.1859 lr 6.00e-04 gnorm 0.11 54,503 tok/s peakVRAM 13.7GB +[19:53:47] step 2200/4768 loss 3.1875 lr 6.00e-04 gnorm 0.10 54,505 tok/s peakVRAM 13.7GB +[19:56:59] step 2220/4768 loss 3.1795 lr 6.00e-04 gnorm 0.11 54,505 tok/s peakVRAM 13.7GB +[20:00:12] step 2240/4768 loss 3.1762 lr 6.00e-04 gnorm 0.10 54,506 tok/s peakVRAM 13.7GB +[20:03:24] step 2260/4768 loss 3.1724 lr 6.00e-04 gnorm 0.10 54,505 tok/s peakVRAM 13.7GB +[20:06:36] step 2280/4768 loss 3.1688 lr 6.00e-04 gnorm 0.09 54,507 tok/s peakVRAM 13.7GB +[20:09:49] step 2300/4768 loss 3.1722 lr 6.00e-04 gnorm 0.11 54,505 tok/s peakVRAM 13.7GB +[20:13:01] step 2320/4768 loss 3.1690 lr 6.00e-04 gnorm 0.11 54,506 tok/s peakVRAM 13.7GB +[20:16:14] step 2340/4768 loss 3.1635 lr 6.00e-04 gnorm 0.10 54,504 tok/s peakVRAM 13.7GB +[20:19:26] step 2360/4768 loss 3.1542 lr 6.00e-04 gnorm 0.10 54,504 tok/s peakVRAM 13.7GB +[20:22:38] step 2380/4768 loss 3.1533 lr 6.00e-04 gnorm 0.09 54,501 tok/s peakVRAM 13.7GB +[20:25:51] step 2400/4768 loss 3.1534 lr 6.00e-04 gnorm 0.09 54,503 tok/s peakVRAM 13.7GB +[20:29:03] step 2420/4768 loss 3.1563 lr 6.00e-04 gnorm 0.09 54,498 tok/s peakVRAM 13.7GB +[20:32:16] step 2440/4768 loss 3.1502 lr 6.00e-04 gnorm 0.10 54,500 tok/s peakVRAM 13.7GB +[20:35:28] step 2460/4768 loss 3.1489 lr 6.00e-04 gnorm 0.10 54,499 tok/s peakVRAM 13.7GB +[20:38:40] step 2480/4768 loss 3.1461 lr 6.00e-04 gnorm 0.10 54,500 tok/s peakVRAM 13.7GB +[20:41:53] step 2500/4768 loss 3.1406 lr 6.00e-04 gnorm 0.10 54,500 tok/s peakVRAM 13.7GB +[20:45:05] step 2520/4768 loss 3.1425 lr 6.00e-04 gnorm 0.10 54,504 tok/s peakVRAM 13.7GB +[20:48:18] step 2540/4768 loss 3.1422 lr 6.00e-04 gnorm 0.09 54,503 tok/s peakVRAM 13.7GB +[20:51:30] step 2560/4768 loss 3.1397 lr 6.00e-04 gnorm 0.10 54,504 tok/s peakVRAM 13.7GB +[20:54:42] step 2580/4768 loss 3.1356 lr 6.00e-04 gnorm 0.10 54,502 tok/s peakVRAM 13.7GB +[20:57:55] step 2600/4768 loss 3.1255 lr 6.00e-04 gnorm 0.10 54,505 tok/s peakVRAM 13.7GB +[21:01:07] step 2620/4768 loss 3.1343 lr 6.00e-04 gnorm 0.10 54,503 tok/s peakVRAM 13.7GB +[21:04:20] step 2640/4768 loss 3.1348 lr 6.00e-04 gnorm 0.10 54,502 tok/s peakVRAM 13.7GB +[21:07:32] step 2660/4768 loss 3.1334 lr 6.00e-04 gnorm 0.11 54,499 tok/s peakVRAM 13.7GB +[21:10:44] step 2680/4768 loss 3.1281 lr 6.00e-04 gnorm 0.10 54,502 tok/s peakVRAM 13.7GB +[21:13:57] step 2700/4768 loss 3.1312 lr 6.00e-04 gnorm 0.10 54,503 tok/s peakVRAM 13.7GB +[21:17:09] step 2720/4768 loss 3.1138 lr 6.00e-04 gnorm 0.09 54,499 tok/s peakVRAM 13.7GB +[21:20:22] step 2740/4768 loss 3.1155 lr 6.00e-04 gnorm 0.10 54,502 tok/s peakVRAM 13.7GB +[21:23:34] step 2760/4768 loss 3.1185 lr 6.00e-04 gnorm 0.09 54,500 tok/s peakVRAM 13.7GB +[21:26:46] step 2780/4768 loss 3.1098 lr 6.00e-04 gnorm 0.11 54,502 tok/s peakVRAM 13.7GB +[21:29:59] step 2800/4768 loss 3.1045 lr 6.00e-04 gnorm 0.11 54,500 tok/s peakVRAM 13.7GB +[21:33:11] step 2820/4768 loss 3.1017 lr 6.00e-04 gnorm 0.09 54,501 tok/s peakVRAM 13.7GB +[21:36:23] step 2840/4768 loss 3.1099 lr 6.00e-04 gnorm 0.10 54,503 tok/s peakVRAM 13.7GB +[21:39:36] step 2860/4768 loss 3.1015 lr 6.00e-04 gnorm 0.10 54,503 tok/s peakVRAM 13.7GB +[21:42:48] step 2880/4768 loss 3.1037 lr 6.00e-04 gnorm 0.10 54,501 tok/s peakVRAM 13.7GB +[21:46:01] step 2900/4768 loss 3.1048 lr 6.00e-04 gnorm 0.09 54,497 tok/s peakVRAM 13.7GB +[21:49:13] step 2920/4768 loss 3.1120 lr 6.00e-04 gnorm 0.10 54,500 tok/s peakVRAM 13.7GB +[21:52:25] step 2940/4768 loss 3.1023 lr 6.00e-04 gnorm 0.09 54,505 tok/s peakVRAM 13.7GB +[21:55:38] step 2960/4768 loss 3.1040 lr 6.00e-04 gnorm 0.10 54,500 tok/s peakVRAM 13.7GB +[21:58:50] step 2980/4768 loss 3.0913 lr 6.00e-04 gnorm 0.10 54,500 tok/s peakVRAM 13.7GB +[22:02:03] step 3000/4768 loss 3.0920 lr 6.00e-04 gnorm 0.09 54,502 tok/s peakVRAM 13.7GB +[22:05:15] step 3020/4768 loss 3.0874 lr 6.00e-04 gnorm 0.11 54,504 tok/s peakVRAM 13.7GB +[22:08:27] step 3040/4768 loss 3.0845 lr 6.00e-04 gnorm 0.09 54,501 tok/s peakVRAM 13.7GB +[22:11:40] step 3060/4768 loss 3.0876 lr 6.00e-04 gnorm 0.09 54,503 tok/s peakVRAM 13.7GB +[22:14:52] step 3080/4768 loss 3.0810 lr 6.00e-04 gnorm 0.09 54,507 tok/s peakVRAM 13.7GB +[22:18:05] step 3100/4768 loss 3.0741 lr 6.00e-04 gnorm 0.11 54,507 tok/s peakVRAM 13.7GB +[22:21:17] step 3120/4768 loss 3.0905 lr 6.00e-04 gnorm 0.10 54,507 tok/s peakVRAM 13.7GB +[22:24:29] step 3140/4768 loss 3.0764 lr 6.00e-04 gnorm 0.09 54,506 tok/s peakVRAM 13.7GB +[22:27:42] step 3160/4768 loss 3.0722 lr 6.00e-04 gnorm 0.10 54,504 tok/s peakVRAM 13.7GB +[22:30:54] step 3180/4768 loss 3.0783 lr 6.00e-04 gnorm 0.10 54,505 tok/s peakVRAM 13.7GB +[22:34:06] step 3200/4768 loss 3.0794 lr 6.00e-04 gnorm 0.09 54,506 tok/s peakVRAM 13.7GB +[22:37:19] step 3220/4768 loss 3.0889 lr 6.00e-04 gnorm 0.09 54,507 tok/s peakVRAM 13.7GB +[22:40:31] step 3240/4768 loss 3.0708 lr 6.00e-04 gnorm 0.09 54,504 tok/s peakVRAM 13.7GB +[22:43:44] step 3260/4768 loss 3.0677 lr 6.00e-04 gnorm 0.09 54,503 tok/s peakVRAM 13.7GB +[22:46:56] step 3280/4768 loss 3.0627 lr 6.00e-04 gnorm 0.09 54,504 tok/s peakVRAM 13.7GB +[22:50:08] step 3300/4768 loss 3.0740 lr 6.00e-04 gnorm 0.09 54,506 tok/s peakVRAM 13.7GB +[22:53:21] step 3320/4768 loss 3.0677 lr 6.00e-04 gnorm 0.09 54,505 tok/s peakVRAM 13.7GB +[22:56:33] step 3340/4768 loss 3.0681 lr 6.00e-04 gnorm 0.09 54,504 tok/s peakVRAM 13.7GB +[22:59:46] step 3360/4768 loss 3.0593 lr 6.00e-04 gnorm 0.10 54,507 tok/s peakVRAM 13.7GB +[23:02:58] step 3380/4768 loss 3.0687 lr 6.00e-04 gnorm 0.10 54,505 tok/s peakVRAM 13.7GB +[23:06:10] step 3400/4768 loss 3.0538 lr 6.00e-04 gnorm 0.09 54,504 tok/s peakVRAM 13.7GB +[23:09:23] step 3420/4768 loss 3.0617 lr 6.00e-04 gnorm 0.09 54,504 tok/s peakVRAM 13.7GB +[23:12:35] step 3440/4768 loss 3.0533 lr 6.00e-04 gnorm 0.10 54,503 tok/s peakVRAM 13.7GB +[23:15:47] step 3460/4768 loss 3.0629 lr 6.00e-04 gnorm 0.09 54,506 tok/s peakVRAM 13.7GB +[23:19:00] step 3480/4768 loss 3.0618 lr 6.00e-04 gnorm 0.08 54,506 tok/s peakVRAM 13.7GB +[23:22:12] step 3500/4768 loss 3.0479 lr 6.00e-04 gnorm 0.09 54,505 tok/s peakVRAM 13.7GB +[23:25:25] step 3520/4768 loss 3.0510 lr 6.00e-04 gnorm 0.09 54,505 tok/s peakVRAM 13.7GB +[23:28:37] step 3540/4768 loss 3.0538 lr 6.00e-04 gnorm 0.10 54,505 tok/s peakVRAM 13.7GB +[23:31:49] step 3560/4768 loss 3.0581 lr 6.00e-04 gnorm 0.09 54,503 tok/s peakVRAM 13.7GB +[23:35:02] step 3580/4768 loss 3.0513 lr 6.00e-04 gnorm 0.08 54,502 tok/s peakVRAM 13.7GB +[23:38:14] step 3600/4768 loss 3.0537 lr 6.00e-04 gnorm 0.08 54,506 tok/s peakVRAM 13.7GB +[23:41:27] step 3620/4768 loss 3.0451 lr 6.00e-04 gnorm 0.09 54,506 tok/s peakVRAM 13.7GB +[23:44:39] step 3640/4768 loss 3.0531 lr 6.00e-04 gnorm 0.09 54,506 tok/s peakVRAM 13.7GB +[23:47:51] step 3660/4768 loss 3.0418 lr 6.00e-04 gnorm 0.09 54,508 tok/s peakVRAM 13.7GB +[23:51:04] step 3680/4768 loss 3.0485 lr 6.00e-04 gnorm 0.09 54,510 tok/s peakVRAM 13.7GB +[23:54:16] step 3700/4768 loss 3.0457 lr 6.00e-04 gnorm 0.09 54,511 tok/s peakVRAM 13.7GB +[23:57:28] step 3720/4768 loss 3.0391 lr 6.00e-04 gnorm 0.09 54,509 tok/s peakVRAM 13.7GB +[00:00:41] step 3740/4768 loss 3.0351 lr 6.00e-04 gnorm 0.09 54,507 tok/s peakVRAM 13.7GB +[00:03:53] step 3760/4768 loss 3.0340 lr 6.00e-04 gnorm 0.09 54,508 tok/s peakVRAM 13.7GB +[00:07:05] step 3780/4768 loss 3.0346 lr 6.00e-04 gnorm 0.09 54,508 tok/s peakVRAM 13.7GB +[00:10:18] step 3800/4768 loss 3.0353 lr 6.00e-04 gnorm 0.08 54,509 tok/s peakVRAM 13.7GB +[00:13:30] step 3820/4768 loss 3.0289 lr 5.61e-04 gnorm 0.09 54,503 tok/s peakVRAM 13.7GB +[00:16:43] step 3840/4768 loss 3.0271 lr 5.13e-04 gnorm 0.08 54,505 tok/s peakVRAM 13.7GB +[00:19:55] step 3860/4768 loss 3.0204 lr 4.83e-04 gnorm 0.07 54,505 tok/s peakVRAM 13.7GB +[00:23:07] step 3880/4768 loss 3.0173 lr 4.59e-04 gnorm 0.07 54,507 tok/s peakVRAM 13.7GB +[00:26:20] step 3900/4768 loss 3.0108 lr 4.39e-04 gnorm 0.08 54,502 tok/s peakVRAM 13.7GB +[00:29:32] step 3920/4768 loss 3.0052 lr 4.21e-04 gnorm 0.08 54,504 tok/s peakVRAM 13.7GB +[00:32:45] step 3940/4768 loss 2.9974 lr 4.05e-04 gnorm 0.07 54,507 tok/s peakVRAM 13.7GB +[00:35:57] step 3960/4768 loss 2.9981 lr 3.89e-04 gnorm 0.08 54,506 tok/s peakVRAM 13.7GB +[00:39:09] step 3980/4768 loss 2.9899 lr 3.75e-04 gnorm 0.08 54,503 tok/s peakVRAM 13.7GB +[00:42:22] step 4000/4768 loss 2.9894 lr 3.62e-04 gnorm 0.07 54,504 tok/s peakVRAM 13.7GB +[00:45:34] step 4020/4768 loss 2.9862 lr 3.50e-04 gnorm 0.07 54,505 tok/s peakVRAM 13.7GB +[00:48:46] step 4040/4768 loss 2.9767 lr 3.38e-04 gnorm 0.07 54,504 tok/s peakVRAM 13.7GB +[00:51:59] step 4060/4768 loss 2.9768 lr 3.26e-04 gnorm 0.07 54,504 tok/s peakVRAM 13.7GB +[00:55:11] step 4080/4768 loss 2.9678 lr 3.15e-04 gnorm 0.07 54,506 tok/s peakVRAM 13.7GB +[00:58:24] step 4100/4768 loss 2.9733 lr 3.05e-04 gnorm 0.07 54,503 tok/s peakVRAM 13.7GB +[01:01:36] step 4120/4768 loss 2.9621 lr 2.95e-04 gnorm 0.07 54,504 tok/s peakVRAM 13.7GB +[01:04:48] step 4140/4768 loss 2.9543 lr 2.85e-04 gnorm 0.07 54,505 tok/s peakVRAM 13.7GB +[01:08:01] step 4160/4768 loss 2.9723 lr 2.75e-04 gnorm 0.07 54,506 tok/s peakVRAM 13.7GB +[01:11:13] step 4180/4768 loss 2.9649 lr 2.66e-04 gnorm 0.07 54,508 tok/s peakVRAM 13.7GB +[01:14:25] step 4200/4768 loss 2.9585 lr 2.57e-04 gnorm 0.07 54,509 tok/s peakVRAM 13.7GB +[01:17:38] step 4220/4768 loss 2.9518 lr 2.48e-04 gnorm 0.07 54,510 tok/s peakVRAM 13.7GB +[01:20:50] step 4240/4768 loss 2.9587 lr 2.40e-04 gnorm 0.07 54,507 tok/s peakVRAM 13.7GB +[01:24:03] step 4260/4768 loss 2.9504 lr 2.31e-04 gnorm 0.07 54,510 tok/s peakVRAM 13.7GB +[01:27:15] step 4280/4768 loss 2.9496 lr 2.23e-04 gnorm 0.07 54,512 tok/s peakVRAM 13.7GB +[01:30:27] step 4300/4768 loss 2.9478 lr 2.15e-04 gnorm 0.07 54,510 tok/s peakVRAM 13.7GB +[01:33:40] step 4320/4768 loss 2.9426 lr 2.07e-04 gnorm 0.07 54,509 tok/s peakVRAM 13.7GB +[01:36:52] step 4340/4768 loss 2.9412 lr 1.99e-04 gnorm 0.07 54,507 tok/s peakVRAM 13.7GB +[01:40:04] step 4360/4768 loss 2.9350 lr 1.92e-04 gnorm 0.07 54,505 tok/s peakVRAM 13.7GB +[01:43:17] step 4380/4768 loss 2.9421 lr 1.84e-04 gnorm 0.07 54,509 tok/s peakVRAM 13.7GB +[01:46:29] step 4400/4768 loss 2.9398 lr 1.77e-04 gnorm 0.07 54,509 tok/s peakVRAM 13.7GB +[01:49:42] step 4420/4768 loss 2.9329 lr 1.70e-04 gnorm 0.07 54,506 tok/s peakVRAM 13.7GB +[01:52:54] step 4440/4768 loss 2.9391 lr 1.63e-04 gnorm 0.07 54,507 tok/s peakVRAM 13.7GB +[01:56:06] step 4460/4768 loss 2.9348 lr 1.56e-04 gnorm 0.07 54,505 tok/s peakVRAM 13.7GB +[01:59:19] step 4480/4768 loss 2.9299 lr 1.49e-04 gnorm 0.07 54,504 tok/s peakVRAM 13.7GB +[02:02:31] step 4500/4768 loss 2.9271 lr 1.42e-04 gnorm 0.07 54,502 tok/s peakVRAM 13.7GB +[02:05:43] step 4520/4768 loss 2.9291 lr 1.36e-04 gnorm 0.07 54,504 tok/s peakVRAM 13.7GB +[02:08:56] step 4540/4768 loss 2.9205 lr 1.29e-04 gnorm 0.07 54,503 tok/s peakVRAM 13.7GB +[02:12:08] step 4560/4768 loss 2.9235 lr 1.23e-04 gnorm 0.07 54,504 tok/s peakVRAM 13.7GB +[02:15:21] step 4580/4768 loss 2.9269 lr 1.16e-04 gnorm 0.07 54,504 tok/s peakVRAM 13.7GB +[02:18:33] step 4600/4768 loss 2.9252 lr 1.10e-04 gnorm 0.07 54,506 tok/s peakVRAM 13.7GB +[02:21:45] step 4620/4768 loss 2.9120 lr 1.04e-04 gnorm 0.07 54,507 tok/s peakVRAM 13.7GB +[02:24:58] step 4640/4768 loss 2.9213 lr 9.78e-05 gnorm 0.07 54,508 tok/s peakVRAM 13.7GB +[02:28:10] step 4660/4768 loss 2.9140 lr 9.18e-05 gnorm 0.07 54,508 tok/s peakVRAM 13.7GB +[02:31:22] step 4680/4768 loss 2.9119 lr 8.58e-05 gnorm 0.07 54,508 tok/s peakVRAM 13.7GB +[02:34:35] step 4700/4768 loss 2.9078 lr 7.99e-05 gnorm 0.07 54,507 tok/s peakVRAM 13.7GB +[02:37:47] step 4720/4768 loss 2.9089 lr 7.41e-05 gnorm 0.07 54,505 tok/s peakVRAM 13.7GB +[02:41:00] step 4740/4768 loss 2.9066 lr 6.83e-05 gnorm 0.07 54,507 tok/s peakVRAM 13.7GB +[02:44:12] step 4760/4768 loss 2.9157 lr 6.26e-05 gnorm 0.07 54,512 tok/s peakVRAM 13.7GB +[02:45:31] ckpt @ 4768 +[02:45:31] DONE diff --git a/R1/run/val_en.json b/R1/run/val_en.json new file mode 100644 index 0000000000000000000000000000000000000000..d6a28349d69597f4b30b610d16b1d28f4b0848b2 --- /dev/null +++ b/R1/run/val_en.json @@ -0,0 +1 @@ +{"schema": "val_bpb_v0", "ckpt_sha256": "c2999a7b90a8fd5a27b23e44e05c793b7359196f667d8118071fe51ae4d43df1", "step": 4768, "val_sha256": "046bc03b426b75f0e84db7cdf3965a9653d94e722b81b0d03fd0cb92884cf0c5", "tokenizer_sha256": "0640d3bd3674d7a6e59c540d945a2ad275a12e1bcaeb6007c37a90751f9815dd", "script_sha256": "ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1", "trainer_sha256": "db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b", "window": 4096, "n_windows": 547, "tokens": 2240512, "bytes": 9799872, "sum_nat": 6787999.6875, "bpb": 0.9993001425643195, "ce_token": 3.029664508603391, "P": 183561103, "dtype": "fp32, tf32 off, bez autocast", "torch": "2.6.0+cu124", "device": "NVIDIA GeForce RTX 4090"} \ No newline at end of file diff --git a/R1/run/val_en.out b/R1/run/val_en.out new file mode 100644 index 0000000000000000000000000000000000000000..ca8c55938e99e1565de9dd90ade66ae5d94c969b --- /dev/null +++ b/R1/run/val_en.out @@ -0,0 +1 @@ +VAL-BPB-OK step 4768 bpb 0.99930 ce 3.02966 bajty 9799872 P 183561103 -> /home/ubuntu/kolektyw/dowody400m/wyniki/R1/val_en.json diff --git a/R1/run/val_pl.json b/R1/run/val_pl.json new file mode 100644 index 0000000000000000000000000000000000000000..0cb97f6c6ce99c9f0c93b24a46daacfa867fd72d --- /dev/null +++ b/R1/run/val_pl.json @@ -0,0 +1 @@ +{"schema": "val_bpb_v0", "ckpt_sha256": "c2999a7b90a8fd5a27b23e44e05c793b7359196f667d8118071fe51ae4d43df1", "step": 4768, "val_sha256": "99e66265f47d249c72f5e76838aa23e82959b99089f720bea6e9ee2cb1f49ee1", "tokenizer_sha256": "0640d3bd3674d7a6e59c540d945a2ad275a12e1bcaeb6007c37a90751f9815dd", "script_sha256": "ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1", "trainer_sha256": "db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b", "window": 4096, "n_windows": 406, "tokens": 1662976, "bytes": 7234678, "sum_nat": 5448028.666015625, "bpb": 1.0864124068938046, "ce_token": 3.276071732854608, "P": 183561103, "dtype": "fp32, tf32 off, bez autocast", "torch": "2.6.0+cu124", "device": "NVIDIA GeForce RTX 4090"} \ No newline at end of file diff --git a/R1/run/val_pl.out b/R1/run/val_pl.out new file mode 100644 index 0000000000000000000000000000000000000000..31008b9ff8505d6b2a6a5f827521e27244e91172 --- /dev/null +++ b/R1/run/val_pl.out @@ -0,0 +1 @@ +VAL-BPB-OK step 4768 bpb 1.08641 ce 3.27607 bajty 7234678 P 183561103 -> /home/ubuntu/kolektyw/dowody400m/wyniki/R1/val_pl.json diff --git a/R1/run/wynik.json b/R1/run/wynik.json new file mode 100644 index 0000000000000000000000000000000000000000..71531ad957f421a5be076398822e6b77a80548fd --- /dev/null +++ b/R1/run/wynik.json @@ -0,0 +1 @@ +{"ramie": "R1", "ckpt_sha256": "c2999a7b90a8fd5a27b23e44e05c793b7359196f667d8118071fe51ae4d43df1", "bpb_pl": 1.0864124068938046, "bpb_en": 0.9993001425643195, "bpb_wazona_40_60": 1.0341450482961134, "val_pl_sha256": "99e66265f47d249c72f5e76838aa23e82959b99089f720bea6e9ee2cb1f49ee1", "val_en_sha256": "046bc03b426b75f0e84db7cdf3965a9653d94e722b81b0d03fd0cb92884cf0c5", "P": 183561103, "step": 4768} diff --git a/R2/run/args.txt b/R2/run/args.txt new file mode 100644 index 0000000000000000000000000000000000000000..8fa8f6782751864444ade515f52a3aec1334a25b --- /dev/null +++ b/R2/run/args.txt @@ -0,0 +1 @@ +--data-dir /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2 --out-dir /home/ubuntu/kolektyw/dowody400m/wyniki/R2 --run-id dowody_R2 --vocab 32768 --dtype uint16 --block 4096 --n-layer 16 --n-embd 896 --n-head 14 --value-residual --batch 128 --grad-accum 8 --norm rmsnorm --pos rope --rope-theta 500000 --ffn swiglu --ffn-mult 2.667 --qk-norm --attn-bias 0 --optimizer muon --muon-lr 0.02 --lr 6e-4 --min-lr 6e-05 --wd 0.1 --schedule wsd --decay-frac 0.2 --warmup 500 --act-ckpt --loss-chunk 1024 --sampler random --seed 2 --data-seed 20261022 --compile --log-every 20 --eval-every 4769 --eval-iters 1 --steps 4768 --ckpt-every 4768 \ No newline at end of file diff --git a/R2/run/metrics.jsonl b/R2/run/metrics.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..04761037074814c4876bf58792dd5960eb3238be --- /dev/null +++ b/R2/run/metrics.jsonl @@ -0,0 +1,238 @@ +{"step": 20, "loss": 10.397884702682495, "lr": 2.3999999999999997e-05, "tok_s": 40238.725610108624} +{"step": 40, "loss": 9.583148860931397, "lr": 4.7999999999999994e-05, "tok_s": 54807.99307010977} +{"step": 60, "loss": 8.702110242843627, "lr": 7.199999999999999e-05, "tok_s": 54817.099398643084} +{"step": 80, "loss": 8.048216319084167, "lr": 9.599999999999999e-05, "tok_s": 54813.46631413231} +{"step": 100, "loss": 7.60359673500061, "lr": 0.00011999999999999999, "tok_s": 54786.64525969034} +{"step": 120, "loss": 7.134544539451599, "lr": 0.00014399999999999998, "tok_s": 54771.8693336949} +{"step": 140, "loss": 6.719778347015381, "lr": 0.000168, "tok_s": 54761.4414950081} +{"step": 160, "loss": 6.357814002037048, "lr": 0.00019199999999999998, "tok_s": 54756.25132274384} +{"step": 180, "loss": 6.088592314720154, "lr": 0.00021599999999999996, "tok_s": 54753.86717044771} +{"step": 200, "loss": 5.867264747619629, "lr": 0.00023999999999999998, "tok_s": 54750.49546345217} +{"step": 220, "loss": 5.679474687576294, "lr": 0.00026399999999999997, "tok_s": 54744.47887142565} +{"step": 240, "loss": 5.506812405586243, "lr": 0.00028799999999999995, "tok_s": 54744.254409518726} +{"step": 260, "loss": 5.342029356956482, "lr": 0.000312, "tok_s": 54745.31397583071} +{"step": 280, "loss": 5.183782267570495, "lr": 0.000336, "tok_s": 54745.73968104227} +{"step": 300, "loss": 5.037600636482239, "lr": 0.00035999999999999997, "tok_s": 54747.28016638277} +{"step": 320, "loss": 4.903152680397033, "lr": 0.00038399999999999996, "tok_s": 54747.2259872589} +{"step": 340, "loss": 4.753053069114685, "lr": 0.000408, "tok_s": 54743.62539447325} +{"step": 360, "loss": 4.627336192131042, "lr": 0.00043199999999999993, "tok_s": 54744.52377765653} +{"step": 380, "loss": 4.507805275917053, "lr": 0.00045599999999999997, "tok_s": 54744.07396910179} +{"step": 400, "loss": 4.406189775466919, "lr": 0.00047999999999999996, "tok_s": 54745.248556695624} +{"step": 420, "loss": 4.3287101030349735, "lr": 0.000504, "tok_s": 54746.63111434638} +{"step": 440, "loss": 4.250546193122863, "lr": 0.0005279999999999999, "tok_s": 54744.49393101838} +{"step": 460, "loss": 4.180587100982666, "lr": 0.000552, "tok_s": 54744.28262047694} +{"step": 480, "loss": 4.133445358276367, "lr": 0.0005759999999999999, "tok_s": 54742.05111447003} +{"step": 500, "loss": 4.077378940582276, "lr": 0.0006, "tok_s": 54735.746861491934} +{"step": 520, "loss": 4.039735519886017, "lr": 0.0006, "tok_s": 54734.59454756114} +{"step": 540, "loss": 3.9894410133361817, "lr": 0.0006, "tok_s": 54735.67090646125} +{"step": 560, "loss": 3.9379420280456543, "lr": 0.0006, "tok_s": 54734.24748686994} +{"step": 580, "loss": 3.8967807412147524, "lr": 0.0006, "tok_s": 54731.84856297202} +{"step": 600, "loss": 3.853356122970581, "lr": 0.0006, "tok_s": 54728.07101962031} +{"step": 620, "loss": 3.830194354057312, "lr": 0.0006, "tok_s": 54729.95077007126} +{"step": 640, "loss": 3.804822540283203, "lr": 0.0006, "tok_s": 54735.461230798064} +{"step": 660, "loss": 3.7740728735923765, "lr": 0.0006, "tok_s": 54729.04871074365} +{"step": 680, "loss": 3.7512762904167176, "lr": 0.0006, "tok_s": 54734.60319859043} +{"step": 700, "loss": 3.7198967695236207, "lr": 0.0006, "tok_s": 54736.26179447402} +{"step": 720, "loss": 3.689478862285614, "lr": 0.0006, "tok_s": 54732.6804213084} +{"step": 740, "loss": 3.685812699794769, "lr": 0.0006, "tok_s": 54731.935541489875} +{"step": 760, "loss": 3.6607274770736695, "lr": 0.0006, "tok_s": 54731.33303191928} +{"step": 780, "loss": 3.6346758127212526, "lr": 0.0006, "tok_s": 54731.053100378515} +{"step": 800, "loss": 3.6206260561943053, "lr": 0.0006, "tok_s": 54732.49985284107} +{"step": 820, "loss": 3.6039608001708983, "lr": 0.0006, "tok_s": 54729.91698913318} +{"step": 840, "loss": 3.5898113369941713, "lr": 0.0006, "tok_s": 54727.83477447694} +{"step": 860, "loss": 3.5693995118141175, "lr": 0.0006, "tok_s": 54730.031476715136} +{"step": 880, "loss": 3.5635278701782225, "lr": 0.0006, "tok_s": 54727.39838350762} +{"step": 900, "loss": 3.5455939054489134, "lr": 0.0006, "tok_s": 54727.977719910945} +{"step": 920, "loss": 3.5379260063171385, "lr": 0.0006, "tok_s": 54731.57727623052} +{"step": 940, "loss": 3.533212733268738, "lr": 0.0006, "tok_s": 54730.919401726045} +{"step": 960, "loss": 3.50755535364151, "lr": 0.0006, "tok_s": 54726.91051644983} +{"step": 980, "loss": 3.504886746406555, "lr": 0.0006, "tok_s": 54728.021849952805} +{"step": 1000, "loss": 3.494468700885773, "lr": 0.0006, "tok_s": 54723.011229484866} +{"step": 1020, "loss": 3.4891278624534605, "lr": 0.0006, "tok_s": 54721.26009162651} +{"step": 1040, "loss": 3.462520432472229, "lr": 0.0006, "tok_s": 54717.87878380706} +{"step": 1060, "loss": 3.473784339427948, "lr": 0.0006, "tok_s": 54717.16643817577} +{"step": 1080, "loss": 3.4629327297210692, "lr": 0.0006, "tok_s": 54721.490424638614} +{"step": 1100, "loss": 3.4472741365432737, "lr": 0.0006, "tok_s": 54723.90764245979} +{"step": 1120, "loss": 3.4296505212783814, "lr": 0.0006, "tok_s": 54723.72039083601} +{"step": 1140, "loss": 3.4227797865867613, "lr": 0.0006, "tok_s": 54728.42780881539} +{"step": 1160, "loss": 3.4173384070396424, "lr": 0.0006, "tok_s": 54723.908800019264} +{"step": 1180, "loss": 3.4195353984832764, "lr": 0.0006, "tok_s": 54724.58325697108} +{"step": 1200, "loss": 3.4017979860305787, "lr": 0.0006, "tok_s": 54724.26737344753} +{"step": 1220, "loss": 3.3993475437164307, "lr": 0.0006, "tok_s": 54726.06548645609} +{"step": 1240, "loss": 3.391629421710968, "lr": 0.0006, "tok_s": 54727.966278800595} +{"step": 1260, "loss": 3.3782228231430054, "lr": 0.0006, "tok_s": 54728.5064678723} +{"step": 1280, "loss": 3.380716252326965, "lr": 0.0006, "tok_s": 54722.33674319356} +{"step": 1300, "loss": 3.373535656929016, "lr": 0.0006, "tok_s": 54718.56738859903} +{"step": 1320, "loss": 3.365945839881897, "lr": 0.0006, "tok_s": 54718.009559518374} +{"step": 1340, "loss": 3.366354763507843, "lr": 0.0006, "tok_s": 54719.10487319138} +{"step": 1360, "loss": 3.357510209083557, "lr": 0.0006, "tok_s": 54716.39161997889} +{"step": 1380, "loss": 3.349139416217804, "lr": 0.0006, "tok_s": 54721.522220687075} +{"step": 1400, "loss": 3.338739049434662, "lr": 0.0006, "tok_s": 54723.850105006284} +{"step": 1420, "loss": 3.326588249206543, "lr": 0.0006, "tok_s": 54724.84085556827} +{"step": 1440, "loss": 3.33370680809021, "lr": 0.0006, "tok_s": 54722.42920661388} +{"step": 1460, "loss": 3.331179511547089, "lr": 0.0006, "tok_s": 54722.33115999262} +{"step": 1480, "loss": 3.33524831533432, "lr": 0.0006, "tok_s": 54718.77713908803} +{"step": 1500, "loss": 3.307989013195038, "lr": 0.0006, "tok_s": 54720.09402221159} +{"step": 1520, "loss": 3.3153640627861023, "lr": 0.0006, "tok_s": 54718.6874111585} +{"step": 1540, "loss": 3.299214768409729, "lr": 0.0006, "tok_s": 54721.35404928674} +{"step": 1560, "loss": 3.3079004168510435, "lr": 0.0006, "tok_s": 54717.641265126316} +{"step": 1580, "loss": 3.296732175350189, "lr": 0.0006, "tok_s": 54719.23122955691} +{"step": 1600, "loss": 3.2852802872657776, "lr": 0.0006, "tok_s": 54716.158606990204} +{"step": 1620, "loss": 3.2858460545539856, "lr": 0.0006, "tok_s": 54718.809680852944} +{"step": 1640, "loss": 3.282657504081726, "lr": 0.0006, "tok_s": 54718.319584186276} +{"step": 1660, "loss": 3.2839358329772947, "lr": 0.0006, "tok_s": 54722.96193272997} +{"step": 1680, "loss": 3.273321044445038, "lr": 0.0006, "tok_s": 54716.57494126232} +{"step": 1700, "loss": 3.2657527327537537, "lr": 0.0006, "tok_s": 54718.044959610204} +{"step": 1720, "loss": 3.267949402332306, "lr": 0.0006, "tok_s": 54712.358045747584} +{"step": 1740, "loss": 3.261772358417511, "lr": 0.0006, "tok_s": 54710.210878688194} +{"step": 1760, "loss": 3.260621118545532, "lr": 0.0006, "tok_s": 54712.09198879873} +{"step": 1780, "loss": 3.247983956336975, "lr": 0.0006, "tok_s": 54715.218542606846} +{"step": 1800, "loss": 3.2524597883224486, "lr": 0.0006, "tok_s": 54716.171336541316} +{"step": 1820, "loss": 3.2466320514678957, "lr": 0.0006, "tok_s": 54717.424988086896} +{"step": 1840, "loss": 3.241079556941986, "lr": 0.0006, "tok_s": 54712.10648607925} +{"step": 1860, "loss": 3.2403503537178038, "lr": 0.0006, "tok_s": 54710.4659599559} +{"step": 1880, "loss": 3.2358384132385254, "lr": 0.0006, "tok_s": 54706.745058173554} +{"step": 1900, "loss": 3.2309120416641237, "lr": 0.0006, "tok_s": 54706.46531004501} +{"step": 1920, "loss": 3.2303597569465636, "lr": 0.0006, "tok_s": 54706.425161557854} +{"step": 1940, "loss": 3.223045599460602, "lr": 0.0006, "tok_s": 54710.422334629184} +{"step": 1960, "loss": 3.2269864439964295, "lr": 0.0006, "tok_s": 54710.97000426783} +{"step": 1980, "loss": 3.222047746181488, "lr": 0.0006, "tok_s": 54713.79271514927} +{"step": 2000, "loss": 3.2153287887573243, "lr": 0.0006, "tok_s": 54721.72770439228} +{"step": 2020, "loss": 3.2151976823806763, "lr": 0.0006, "tok_s": 54719.691795279694} +{"step": 2040, "loss": 3.2076790571212768, "lr": 0.0006, "tok_s": 54721.02594781771} +{"step": 2060, "loss": 3.207830536365509, "lr": 0.0006, "tok_s": 54716.8153097971} +{"step": 2080, "loss": 3.19669691324234, "lr": 0.0006, "tok_s": 54716.61993787353} +{"step": 2100, "loss": 3.205078649520874, "lr": 0.0006, "tok_s": 54717.88620417166} +{"step": 2120, "loss": 3.1950955986976624, "lr": 0.0006, "tok_s": 54716.531510412766} +{"step": 2140, "loss": 3.20761091709137, "lr": 0.0006, "tok_s": 54717.355959514505} +{"step": 2160, "loss": 3.186537528038025, "lr": 0.0006, "tok_s": 54714.34011192798} +{"step": 2180, "loss": 3.1880959272384644, "lr": 0.0006, "tok_s": 54716.86922452477} +{"step": 2200, "loss": 3.189212453365326, "lr": 0.0006, "tok_s": 54710.697291049124} +{"step": 2220, "loss": 3.182005798816681, "lr": 0.0006, "tok_s": 54709.39535600489} +{"step": 2240, "loss": 3.17874618768692, "lr": 0.0006, "tok_s": 54712.08252813624} +{"step": 2260, "loss": 3.1753636479377745, "lr": 0.0006, "tok_s": 54714.358081866165} +{"step": 2280, "loss": 3.171137976646423, "lr": 0.0006, "tok_s": 54711.677152031705} +{"step": 2300, "loss": 3.1747127294540407, "lr": 0.0006, "tok_s": 54715.94526876368} +{"step": 2320, "loss": 3.1717222332954407, "lr": 0.0006, "tok_s": 54717.84045663478} +{"step": 2340, "loss": 3.1666620850563048, "lr": 0.0006, "tok_s": 54715.35461509705} +{"step": 2360, "loss": 3.157251071929932, "lr": 0.0006, "tok_s": 54718.45662519537} +{"step": 2380, "loss": 3.155860257148743, "lr": 0.0006, "tok_s": 54714.01481724696} +{"step": 2400, "loss": 3.1565531134605407, "lr": 0.0006, "tok_s": 54718.36880449952} +{"step": 2420, "loss": 3.1596901655197143, "lr": 0.0006, "tok_s": 54720.08728207041} +{"step": 2440, "loss": 3.154794490337372, "lr": 0.0006, "tok_s": 54721.31673852506} +{"step": 2460, "loss": 3.1532232761383057, "lr": 0.0006, "tok_s": 54719.27493701316} +{"step": 2480, "loss": 3.150437569618225, "lr": 0.0006, "tok_s": 54716.898496454254} +{"step": 2500, "loss": 3.1446684837341308, "lr": 0.0006, "tok_s": 54716.010073307494} +{"step": 2520, "loss": 3.1466462135314943, "lr": 0.0006, "tok_s": 54718.5767153474} +{"step": 2540, "loss": 3.146539342403412, "lr": 0.0006, "tok_s": 54717.183729220495} +{"step": 2560, "loss": 3.1434460043907166, "lr": 0.0006, "tok_s": 54718.556019501295} +{"step": 2580, "loss": 3.1385318756103517, "lr": 0.0006, "tok_s": 54718.684415694} +{"step": 2600, "loss": 3.1294031739234924, "lr": 0.0006, "tok_s": 54716.10578281999} +{"step": 2620, "loss": 3.137481212615967, "lr": 0.0006, "tok_s": 54711.76957943284} +{"step": 2640, "loss": 3.138892638683319, "lr": 0.0006, "tok_s": 54714.37680056432} +{"step": 2660, "loss": 3.137242865562439, "lr": 0.0006, "tok_s": 54714.25250864838} +{"step": 2680, "loss": 3.1325737953186037, "lr": 0.0006, "tok_s": 54715.855073619445} +{"step": 2700, "loss": 3.135154151916504, "lr": 0.0006, "tok_s": 54717.64630276108} +{"step": 2720, "loss": 3.117984914779663, "lr": 0.0006, "tok_s": 54717.44942723859} +{"step": 2740, "loss": 3.119256818294525, "lr": 0.0006, "tok_s": 54723.33411180745} +{"step": 2760, "loss": 3.1220457315444947, "lr": 0.0006, "tok_s": 54724.06779468343} +{"step": 2780, "loss": 3.1131870627403258, "lr": 0.0006, "tok_s": 54721.67309922795} +{"step": 2800, "loss": 3.108135437965393, "lr": 0.0006, "tok_s": 54720.03581195619} +{"step": 2820, "loss": 3.105840790271759, "lr": 0.0006, "tok_s": 54719.065659265565} +{"step": 2840, "loss": 3.1134299874305724, "lr": 0.0006, "tok_s": 54719.31775949517} +{"step": 2860, "loss": 3.1054877400398255, "lr": 0.0006, "tok_s": 54717.18046162066} +{"step": 2880, "loss": 3.1071001291275024, "lr": 0.0006, "tok_s": 54713.99916181478} +{"step": 2900, "loss": 3.1088037490844727, "lr": 0.0006, "tok_s": 54712.77173575169} +{"step": 2920, "loss": 3.115924334526062, "lr": 0.0006, "tok_s": 54717.879396497636} +{"step": 2940, "loss": 3.10650497674942, "lr": 0.0006, "tok_s": 54719.886780650384} +{"step": 2960, "loss": 3.1084991097450256, "lr": 0.0006, "tok_s": 54720.022127457996} +{"step": 2980, "loss": 3.095391809940338, "lr": 0.0006, "tok_s": 54721.01008411949} +{"step": 3000, "loss": 3.095682442188263, "lr": 0.0006, "tok_s": 54721.42118155174} +{"step": 3020, "loss": 3.0919813394546507, "lr": 0.0006, "tok_s": 54720.8383074097} +{"step": 3040, "loss": 3.0889034152030943, "lr": 0.0006, "tok_s": 54713.81034439485} +{"step": 3060, "loss": 3.092266011238098, "lr": 0.0006, "tok_s": 54717.98729833024} +{"step": 3080, "loss": 3.0846786975860594, "lr": 0.0006, "tok_s": 54720.33380892033} +{"step": 3100, "loss": 3.0786309003829957, "lr": 0.0006, "tok_s": 54724.73435667363} +{"step": 3120, "loss": 3.0947417974472047, "lr": 0.0006, "tok_s": 54720.54507078586} +{"step": 3140, "loss": 3.0798319816589355, "lr": 0.0006, "tok_s": 54724.353374571554} +{"step": 3160, "loss": 3.0761234402656554, "lr": 0.0006, "tok_s": 54722.32339798344} +{"step": 3180, "loss": 3.082308828830719, "lr": 0.0006, "tok_s": 54718.39875881372} +{"step": 3200, "loss": 3.0831666231155395, "lr": 0.0006, "tok_s": 54720.005175029415} +{"step": 3220, "loss": 3.0926347255706785, "lr": 0.0006, "tok_s": 54721.79946745477} +{"step": 3240, "loss": 3.0746402978897094, "lr": 0.0006, "tok_s": 54719.029168579524} +{"step": 3260, "loss": 3.071154201030731, "lr": 0.0006, "tok_s": 54716.623069259214} +{"step": 3280, "loss": 3.06698979139328, "lr": 0.0006, "tok_s": 54718.69204051337} +{"step": 3300, "loss": 3.0780227899551393, "lr": 0.0006, "tok_s": 54712.43155387962} +{"step": 3320, "loss": 3.071763741970062, "lr": 0.0006, "tok_s": 54712.519559708606} +{"step": 3340, "loss": 3.0722206830978394, "lr": 0.0006, "tok_s": 54715.91164125627} +{"step": 3360, "loss": 3.0630337476730345, "lr": 0.0006, "tok_s": 54717.922761410126} +{"step": 3380, "loss": 3.0727545022964478, "lr": 0.0006, "tok_s": 54714.51457057679} +{"step": 3400, "loss": 3.058015322685242, "lr": 0.0006, "tok_s": 54718.504279954526} +{"step": 3420, "loss": 3.0660104870796205, "lr": 0.0006, "tok_s": 54717.33254157685} +{"step": 3440, "loss": 3.0576372504234315, "lr": 0.0006, "tok_s": 54715.9172912192} +{"step": 3460, "loss": 3.0667299389839173, "lr": 0.0006, "tok_s": 54714.89221644115} +{"step": 3480, "loss": 3.0667102575302123, "lr": 0.0006, "tok_s": 54716.037710586104} +{"step": 3500, "loss": 3.0526495456695555, "lr": 0.0006, "tok_s": 54713.767462465934} +{"step": 3520, "loss": 3.0552218317985536, "lr": 0.0006, "tok_s": 54717.21783481713} +{"step": 3540, "loss": 3.057752025127411, "lr": 0.0006, "tok_s": 54716.22273131326} +{"step": 3560, "loss": 3.0616055846214296, "lr": 0.0006, "tok_s": 54717.16228560366} +{"step": 3580, "loss": 3.0557958483695984, "lr": 0.0006, "tok_s": 54721.54659541305} +{"step": 3600, "loss": 3.057619869709015, "lr": 0.0006, "tok_s": 54721.054815687006} +{"step": 3620, "loss": 3.0496942520141603, "lr": 0.0006, "tok_s": 54719.45923074687} +{"step": 3640, "loss": 3.0573796391487122, "lr": 0.0006, "tok_s": 54721.89206523288} +{"step": 3660, "loss": 3.046215331554413, "lr": 0.0006, "tok_s": 54720.4641197848} +{"step": 3680, "loss": 3.0530635356903075, "lr": 0.0006, "tok_s": 54724.27445508174} +{"step": 3700, "loss": 3.050299286842346, "lr": 0.0006, "tok_s": 54726.45343839958} +{"step": 3720, "loss": 3.0435306072235107, "lr": 0.0006, "tok_s": 54724.378909399136} +{"step": 3740, "loss": 3.0399710059165956, "lr": 0.0006, "tok_s": 54727.211312294094} +{"step": 3760, "loss": 3.0383835315704344, "lr": 0.0006, "tok_s": 54725.30948286721} +{"step": 3780, "loss": 3.038952612876892, "lr": 0.0006, "tok_s": 54721.125896136524} +{"step": 3800, "loss": 3.040010416507721, "lr": 0.0006, "tok_s": 54717.79967876089} +{"step": 3820, "loss": 3.0331411719322205, "lr": 0.0005609064981513237, "tok_s": 54717.83038129468} +{"step": 3840, "loss": 3.031576919555664, "lr": 0.0005125842723878462, "tok_s": 54716.754111264} +{"step": 3860, "loss": 3.0244616866111755, "lr": 0.0004827194944539712, "tok_s": 54718.43586136199} +{"step": 3880, "loss": 3.0217150330543516, "lr": 0.0004590463745471515, "tok_s": 54721.24361502659} +{"step": 3900, "loss": 3.015034055709839, "lr": 0.0004388133626026285, "tok_s": 54716.04574310089} +{"step": 3920, "loss": 3.009331476688385, "lr": 0.000420851068597545, "tok_s": 54714.75396804418} +{"step": 3940, "loss": 3.001947045326233, "lr": 0.00040453249075661876, "tok_s": 54714.83483408226} +{"step": 3960, "loss": 3.0024040579795837, "lr": 0.00038947504965686095, "tok_s": 54715.71600350052} +{"step": 3980, "loss": 2.994537925720215, "lr": 0.000375424929557729, "tok_s": 54720.73148367389} +{"step": 4000, "loss": 2.9933198213577272, "lr": 0.00036220351177672794, "tok_s": 54722.08182357052} +{"step": 4020, "loss": 2.9906443119049073, "lr": 0.0003496794507866152, "tok_s": 54713.006217316004} +{"step": 4040, "loss": 2.981214869022369, "lr": 0.0003377528171635386, "tok_s": 54714.00338197388} +{"step": 4060, "loss": 2.9809293031692503, "lr": 0.00032634548705926626, "tok_s": 54719.15021436317} +{"step": 4080, "loss": 2.972481036186218, "lr": 0.00031539501058484577, "tok_s": 54714.52505310615} +{"step": 4100, "loss": 2.977624309062958, "lr": 0.00030485053354754334, "tok_s": 54715.570126989114} +{"step": 4120, "loss": 2.96641982793808, "lr": 0.00029466998985632013, "tok_s": 54720.254764858255} +{"step": 4140, "loss": 2.9586235880851746, "lr": 0.0002848181118123862, "tok_s": 54715.854529045886} +{"step": 4160, "loss": 2.9767950534820558, "lr": 0.000275264984658754, "tok_s": 54709.52575088879} +{"step": 4180, "loss": 2.9694134473800657, "lr": 0.0002659849737873323, "tok_s": 54711.81463620257} +{"step": 4200, "loss": 2.963054656982422, "lr": 0.0002569559134994179, "tok_s": 54715.08110939802} +{"step": 4220, "loss": 2.956114983558655, "lr": 0.00024815848336191376, "tok_s": 54715.96895780181} +{"step": 4240, "loss": 2.9624460101127625, "lr": 0.00023957572171486765, "tok_s": 54717.69756415137} +{"step": 4260, "loss": 2.954618144035339, "lr": 0.0002311926411735683, "tok_s": 54720.05290056804} +{"step": 4280, "loss": 2.9539708614349367, "lr": 0.0002229959211471338, "tok_s": 54724.793530241026} +{"step": 4300, "loss": 2.9517720341682434, "lr": 0.00021497365931813446, "tok_s": 54721.169061940236} +{"step": 4320, "loss": 2.9469788312911986, "lr": 0.00020711516882674415, "tok_s": 54717.55419588281} +{"step": 4340, "loss": 2.945694601535797, "lr": 0.00019941081128766396, "tok_s": 54716.70019676318} +{"step": 4360, "loss": 2.939618408679962, "lr": 0.00019185185819315875, "tok_s": 54716.639134634876} +{"step": 4380, "loss": 2.946542811393738, "lr": 0.00018443037501815989, "tok_s": 54713.43659320793} +{"step": 4400, "loss": 2.943904232978821, "lr": 0.00017713912364145464, "tok_s": 54710.85961191617} +{"step": 4420, "loss": 2.9370927572250367, "lr": 0.00016997147966456131, "tok_s": 54713.706474949475} +{"step": 4440, "loss": 2.9435221195220946, "lr": 0.00016292136193923112, "tok_s": 54713.4350276974} +{"step": 4460, "loss": 2.9390411615371703, "lr": 0.00015598317216998812, "tok_s": 54716.44492121314} +{"step": 4480, "loss": 2.9340861797332765, "lr": 0.00014915174288526275, "tok_s": 54719.165055784084} +{"step": 4500, "loss": 2.9313199520111084, "lr": 0.00014242229240208165, "tok_s": 54719.58014227132} +{"step": 4520, "loss": 2.9331663250923157, "lr": 0.00013579038566856907, "tok_s": 54717.37767558506} +{"step": 4540, "loss": 2.9246115565299986, "lr": 0.00012925190007297358, "tok_s": 54717.161536779255} +{"step": 4560, "loss": 2.9280313849449158, "lr": 0.0001228029954703451, "tok_s": 54711.65305834222} +{"step": 4580, "loss": 2.9309247732162476, "lr": 0.00011644008780788571, "tok_s": 54709.7997455041} +{"step": 4600, "loss": 2.9298508882522585, "lr": 0.00011015982583457378, "tok_s": 54707.8044659755} +{"step": 4620, "loss": 2.916302442550659, "lr": 0.00010395907046536478, "tok_s": 54711.310780508815} +{"step": 4640, "loss": 2.9256266355514526, "lr": 9.783487643927846e-05, "tok_s": 54714.35134313796} +{"step": 4660, "loss": 2.9183580040931703, "lr": 9.178447596720887e-05, "tok_s": 54712.94945149312} +{"step": 4680, "loss": 2.9159430027008058, "lr": 8.580526411184025e-05, "tok_s": 54710.84729319185} +{"step": 4700, "loss": 2.9120609164237976, "lr": 7.98947856805662e-05, "tok_s": 54712.34722373372} +{"step": 4720, "loss": 2.9131085753440855, "lr": 7.405072344432876e-05, "tok_s": 54714.52791197849} +{"step": 4740, "loss": 2.9110165357589723, "lr": 6.827088752203553e-05, "tok_s": 54717.93188370816} +{"step": 4760, "loss": 2.920218658447266, "lr": 6.255320579263513e-05, "tok_s": 54719.83497050536} diff --git a/R2/run/srodowisko.txt b/R2/run/srodowisko.txt new file mode 100644 index 0000000000000000000000000000000000000000..0b482d1cd3d82732e78b2ae6d378e22f9b2af4b3 --- /dev/null +++ b/R2/run/srodowisko.txt @@ -0,0 +1,63 @@ +data 2026-10-22T11:59:50Z ARM R2 GPU 1 MINI 0 DATA_SEED 20261022 +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +a607076bd6e11cad879b8213b37492a4b4f5cbf8fa96205b7ebaeace8a25213e /home/ubuntu/kolektyw/dowody400m/kod/ramie.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 diff --git a/R2/run/srodowisko_R2_mini0.txt b/R2/run/srodowisko_R2_mini0.txt new file mode 100644 index 0000000000000000000000000000000000000000..0b482d1cd3d82732e78b2ae6d378e22f9b2af4b3 --- /dev/null +++ b/R2/run/srodowisko_R2_mini0.txt @@ -0,0 +1,63 @@ +data 2026-10-22T11:59:50Z ARM R2 GPU 1 MINI 0 DATA_SEED 20261022 +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +a607076bd6e11cad879b8213b37492a4b4f5cbf8fa96205b7ebaeace8a25213e /home/ubuntu/kolektyw/dowody400m/kod/ramie.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 diff --git a/R2/run/train.log b/R2/run/train.log new file mode 100644 index 0000000000000000000000000000000000000000..260766e962072532469d66075125d8b6454dc8f2 --- /dev/null +++ b/R2/run/train.log @@ -0,0 +1,248 @@ +[13:59:56] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[13:59:56] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[13:59:58] model GPT-ref: 183.6M param (L16 d896 h14) +[13:59:59] optimizer=muon muon_lr=0.02 (mult=33.3x) +[13:59:59] r11 pamiec: act_ckpt=True loss_chunk=1024 +[13:59:59] r13: mixer_layout=AAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[13:59:59] torch.compile enabled +[13:59:59] START gpt-ref: steps=4768 (od 0) lr=0.0006->6e-05 +[14:04:20] step 20/4768 loss 10.3979 lr 2.40e-05 gnorm 2.60 40,239 tok/s peakVRAM 13.7GB +[14:07:31] step 40/4768 loss 9.5831 lr 4.80e-05 gnorm 1.42 54,808 tok/s peakVRAM 13.7GB +[14:10:43] step 60/4768 loss 8.7021 lr 7.20e-05 gnorm 0.76 54,817 tok/s peakVRAM 13.7GB +[14:13:54] step 80/4768 loss 8.0482 lr 9.60e-05 gnorm 1.57 54,813 tok/s peakVRAM 13.7GB +[14:17:05] step 100/4768 loss 7.6036 lr 1.20e-04 gnorm 0.99 54,787 tok/s peakVRAM 13.7GB +[14:20:17] step 120/4768 loss 7.1345 lr 1.44e-04 gnorm 0.74 54,772 tok/s peakVRAM 13.7GB +[14:23:28] step 140/4768 loss 6.7198 lr 1.68e-04 gnorm 0.81 54,761 tok/s peakVRAM 13.7GB +[14:26:40] step 160/4768 loss 6.3578 lr 1.92e-04 gnorm 0.82 54,756 tok/s peakVRAM 13.7GB +[14:29:51] step 180/4768 loss 6.0886 lr 2.16e-04 gnorm 0.72 54,754 tok/s peakVRAM 13.7GB +[14:33:03] step 200/4768 loss 5.8673 lr 2.40e-04 gnorm 0.70 54,750 tok/s peakVRAM 13.7GB +[14:36:14] step 220/4768 loss 5.6795 lr 2.64e-04 gnorm 0.63 54,744 tok/s peakVRAM 13.7GB +[14:39:26] step 240/4768 loss 5.5068 lr 2.88e-04 gnorm 0.73 54,744 tok/s peakVRAM 13.7GB +[14:42:37] step 260/4768 loss 5.3420 lr 3.12e-04 gnorm 0.89 54,745 tok/s peakVRAM 13.7GB +[14:45:49] step 280/4768 loss 5.1838 lr 3.36e-04 gnorm 0.74 54,746 tok/s peakVRAM 13.7GB +[14:49:00] step 300/4768 loss 5.0376 lr 3.60e-04 gnorm 0.77 54,747 tok/s peakVRAM 13.7GB +[14:52:12] step 320/4768 loss 4.9032 lr 3.84e-04 gnorm 0.87 54,747 tok/s peakVRAM 13.7GB +[14:55:24] step 340/4768 loss 4.7531 lr 4.08e-04 gnorm 0.75 54,744 tok/s peakVRAM 13.7GB +[14:58:35] step 360/4768 loss 4.6273 lr 4.32e-04 gnorm 0.59 54,745 tok/s peakVRAM 13.7GB +[15:01:47] step 380/4768 loss 4.5078 lr 4.56e-04 gnorm 0.56 54,744 tok/s peakVRAM 13.7GB +[15:04:58] step 400/4768 loss 4.4062 lr 4.80e-04 gnorm 0.45 54,745 tok/s peakVRAM 13.7GB +[15:08:10] step 420/4768 loss 4.3287 lr 5.04e-04 gnorm 0.46 54,747 tok/s peakVRAM 13.7GB +[15:11:21] step 440/4768 loss 4.2505 lr 5.28e-04 gnorm 0.40 54,744 tok/s peakVRAM 13.7GB +[15:14:33] step 460/4768 loss 4.1806 lr 5.52e-04 gnorm 0.39 54,744 tok/s peakVRAM 13.7GB +[15:17:44] step 480/4768 loss 4.1334 lr 5.76e-04 gnorm 0.35 54,742 tok/s peakVRAM 13.7GB +[15:20:56] step 500/4768 loss 4.0774 lr 6.00e-04 gnorm 0.31 54,736 tok/s peakVRAM 13.7GB +[15:24:07] step 520/4768 loss 4.0397 lr 6.00e-04 gnorm 0.27 54,735 tok/s peakVRAM 13.7GB +[15:27:19] step 540/4768 loss 3.9894 lr 6.00e-04 gnorm 0.22 54,736 tok/s peakVRAM 13.7GB +[15:30:31] step 560/4768 loss 3.9379 lr 6.00e-04 gnorm 0.23 54,734 tok/s peakVRAM 13.7GB +[15:33:42] step 580/4768 loss 3.8968 lr 6.00e-04 gnorm 0.23 54,732 tok/s peakVRAM 13.7GB +[15:36:54] step 600/4768 loss 3.8534 lr 6.00e-04 gnorm 0.21 54,728 tok/s peakVRAM 13.7GB +[15:40:05] step 620/4768 loss 3.8302 lr 6.00e-04 gnorm 0.21 54,730 tok/s peakVRAM 13.7GB +[15:43:17] step 640/4768 loss 3.8048 lr 6.00e-04 gnorm 0.19 54,735 tok/s peakVRAM 13.7GB +[15:46:29] step 660/4768 loss 3.7741 lr 6.00e-04 gnorm 0.17 54,729 tok/s peakVRAM 13.7GB +[15:49:40] step 680/4768 loss 3.7513 lr 6.00e-04 gnorm 0.18 54,735 tok/s peakVRAM 13.7GB +[15:52:52] step 700/4768 loss 3.7199 lr 6.00e-04 gnorm 0.18 54,736 tok/s peakVRAM 13.7GB +[15:56:03] step 720/4768 loss 3.6895 lr 6.00e-04 gnorm 0.17 54,733 tok/s peakVRAM 13.7GB +[15:59:15] step 740/4768 loss 3.6858 lr 6.00e-04 gnorm 0.19 54,732 tok/s peakVRAM 13.7GB +[16:02:26] step 760/4768 loss 3.6607 lr 6.00e-04 gnorm 0.16 54,731 tok/s peakVRAM 13.7GB +[16:05:38] step 780/4768 loss 3.6347 lr 6.00e-04 gnorm 0.16 54,731 tok/s peakVRAM 13.7GB +[16:08:50] step 800/4768 loss 3.6206 lr 6.00e-04 gnorm 0.14 54,732 tok/s peakVRAM 13.7GB +[16:12:01] step 820/4768 loss 3.6040 lr 6.00e-04 gnorm 0.14 54,730 tok/s peakVRAM 13.7GB +[16:15:13] step 840/4768 loss 3.5898 lr 6.00e-04 gnorm 0.15 54,728 tok/s peakVRAM 13.7GB +[16:18:24] step 860/4768 loss 3.5694 lr 6.00e-04 gnorm 0.16 54,730 tok/s peakVRAM 13.7GB +[16:21:36] step 880/4768 loss 3.5635 lr 6.00e-04 gnorm 0.14 54,727 tok/s peakVRAM 13.7GB +[16:24:48] step 900/4768 loss 3.5456 lr 6.00e-04 gnorm 0.14 54,728 tok/s peakVRAM 13.7GB +[16:27:59] step 920/4768 loss 3.5379 lr 6.00e-04 gnorm 0.14 54,732 tok/s peakVRAM 13.7GB +[16:31:11] step 940/4768 loss 3.5332 lr 6.00e-04 gnorm 0.14 54,731 tok/s peakVRAM 13.7GB +[16:34:22] step 960/4768 loss 3.5076 lr 6.00e-04 gnorm 0.13 54,727 tok/s peakVRAM 13.7GB +[16:37:34] step 980/4768 loss 3.5049 lr 6.00e-04 gnorm 0.12 54,728 tok/s peakVRAM 13.7GB +[16:40:46] step 1000/4768 loss 3.4945 lr 6.00e-04 gnorm 0.14 54,723 tok/s peakVRAM 13.7GB +[16:43:57] step 1020/4768 loss 3.4891 lr 6.00e-04 gnorm 0.13 54,721 tok/s peakVRAM 13.7GB +[16:47:09] step 1040/4768 loss 3.4625 lr 6.00e-04 gnorm 0.13 54,718 tok/s peakVRAM 13.7GB +[16:50:20] step 1060/4768 loss 3.4738 lr 6.00e-04 gnorm 0.14 54,717 tok/s peakVRAM 13.7GB +[16:53:32] step 1080/4768 loss 3.4629 lr 6.00e-04 gnorm 0.12 54,721 tok/s peakVRAM 13.7GB +[16:56:44] step 1100/4768 loss 3.4473 lr 6.00e-04 gnorm 0.14 54,724 tok/s peakVRAM 13.7GB +[16:59:55] step 1120/4768 loss 3.4297 lr 6.00e-04 gnorm 0.12 54,724 tok/s peakVRAM 13.7GB +[17:03:07] step 1140/4768 loss 3.4228 lr 6.00e-04 gnorm 0.12 54,728 tok/s peakVRAM 13.7GB +[17:06:19] step 1160/4768 loss 3.4173 lr 6.00e-04 gnorm 0.12 54,724 tok/s peakVRAM 13.7GB +[17:09:30] step 1180/4768 loss 3.4195 lr 6.00e-04 gnorm 0.12 54,725 tok/s peakVRAM 13.7GB +[17:12:42] step 1200/4768 loss 3.4018 lr 6.00e-04 gnorm 0.13 54,724 tok/s peakVRAM 13.7GB +[17:15:53] step 1220/4768 loss 3.3993 lr 6.00e-04 gnorm 0.13 54,726 tok/s peakVRAM 13.7GB +[17:19:05] step 1240/4768 loss 3.3916 lr 6.00e-04 gnorm 0.12 54,728 tok/s peakVRAM 13.7GB +[17:22:17] step 1260/4768 loss 3.3782 lr 6.00e-04 gnorm 0.12 54,729 tok/s peakVRAM 13.7GB +[17:25:28] step 1280/4768 loss 3.3807 lr 6.00e-04 gnorm 0.12 54,722 tok/s peakVRAM 13.7GB +[17:28:40] step 1300/4768 loss 3.3735 lr 6.00e-04 gnorm 0.11 54,719 tok/s peakVRAM 13.7GB +[17:31:51] step 1320/4768 loss 3.3659 lr 6.00e-04 gnorm 0.11 54,718 tok/s peakVRAM 13.7GB +[17:35:03] step 1340/4768 loss 3.3664 lr 6.00e-04 gnorm 0.13 54,719 tok/s peakVRAM 13.7GB +[17:38:15] step 1360/4768 loss 3.3575 lr 6.00e-04 gnorm 0.13 54,716 tok/s peakVRAM 13.7GB +[17:41:26] step 1380/4768 loss 3.3491 lr 6.00e-04 gnorm 0.11 54,722 tok/s peakVRAM 13.7GB +[17:44:38] step 1400/4768 loss 3.3387 lr 6.00e-04 gnorm 0.12 54,724 tok/s peakVRAM 13.7GB +[17:47:50] step 1420/4768 loss 3.3266 lr 6.00e-04 gnorm 0.12 54,725 tok/s peakVRAM 13.7GB +[17:51:01] step 1440/4768 loss 3.3337 lr 6.00e-04 gnorm 0.11 54,722 tok/s peakVRAM 13.7GB +[17:54:13] step 1460/4768 loss 3.3312 lr 6.00e-04 gnorm 0.11 54,722 tok/s peakVRAM 13.7GB +[17:57:24] step 1480/4768 loss 3.3352 lr 6.00e-04 gnorm 0.11 54,719 tok/s peakVRAM 13.7GB +[18:00:36] step 1500/4768 loss 3.3080 lr 6.00e-04 gnorm 0.10 54,720 tok/s peakVRAM 13.7GB +[18:03:48] step 1520/4768 loss 3.3154 lr 6.00e-04 gnorm 0.12 54,719 tok/s peakVRAM 13.7GB +[18:06:59] step 1540/4768 loss 3.2992 lr 6.00e-04 gnorm 0.11 54,721 tok/s peakVRAM 13.7GB +[18:10:11] step 1560/4768 loss 3.3079 lr 6.00e-04 gnorm 0.13 54,718 tok/s peakVRAM 13.7GB +[18:13:23] step 1580/4768 loss 3.2967 lr 6.00e-04 gnorm 0.11 54,719 tok/s peakVRAM 13.7GB +[18:16:34] step 1600/4768 loss 3.2853 lr 6.00e-04 gnorm 0.11 54,716 tok/s peakVRAM 13.7GB +[18:19:46] step 1620/4768 loss 3.2858 lr 6.00e-04 gnorm 0.12 54,719 tok/s peakVRAM 13.7GB +[18:22:57] step 1640/4768 loss 3.2827 lr 6.00e-04 gnorm 0.12 54,718 tok/s peakVRAM 13.7GB +[18:26:09] step 1660/4768 loss 3.2839 lr 6.00e-04 gnorm 0.11 54,723 tok/s peakVRAM 13.7GB +[18:29:21] step 1680/4768 loss 3.2733 lr 6.00e-04 gnorm 0.11 54,717 tok/s peakVRAM 13.7GB +[18:32:32] step 1700/4768 loss 3.2658 lr 6.00e-04 gnorm 0.12 54,718 tok/s peakVRAM 13.7GB +[18:35:44] step 1720/4768 loss 3.2679 lr 6.00e-04 gnorm 0.12 54,712 tok/s peakVRAM 13.7GB +[18:38:56] step 1740/4768 loss 3.2618 lr 6.00e-04 gnorm 0.10 54,710 tok/s peakVRAM 13.7GB +[18:42:07] step 1760/4768 loss 3.2606 lr 6.00e-04 gnorm 0.11 54,712 tok/s peakVRAM 13.7GB +[18:45:19] step 1780/4768 loss 3.2480 lr 6.00e-04 gnorm 0.11 54,715 tok/s peakVRAM 13.7GB +[18:48:31] step 1800/4768 loss 3.2525 lr 6.00e-04 gnorm 0.12 54,716 tok/s peakVRAM 13.7GB +[18:51:42] step 1820/4768 loss 3.2466 lr 6.00e-04 gnorm 0.11 54,717 tok/s peakVRAM 13.7GB +[18:54:54] step 1840/4768 loss 3.2411 lr 6.00e-04 gnorm 0.10 54,712 tok/s peakVRAM 13.7GB +[18:58:06] step 1860/4768 loss 3.2404 lr 6.00e-04 gnorm 0.12 54,710 tok/s peakVRAM 13.7GB +[19:01:17] step 1880/4768 loss 3.2358 lr 6.00e-04 gnorm 0.11 54,707 tok/s peakVRAM 13.7GB +[19:04:29] step 1900/4768 loss 3.2309 lr 6.00e-04 gnorm 0.10 54,706 tok/s peakVRAM 13.7GB +[19:07:41] step 1920/4768 loss 3.2304 lr 6.00e-04 gnorm 0.10 54,706 tok/s peakVRAM 13.7GB +[19:10:52] step 1940/4768 loss 3.2230 lr 6.00e-04 gnorm 0.12 54,710 tok/s peakVRAM 13.7GB +[19:14:04] step 1960/4768 loss 3.2270 lr 6.00e-04 gnorm 0.11 54,711 tok/s peakVRAM 13.7GB +[19:17:16] step 1980/4768 loss 3.2220 lr 6.00e-04 gnorm 0.09 54,714 tok/s peakVRAM 13.7GB +[19:20:27] step 2000/4768 loss 3.2153 lr 6.00e-04 gnorm 0.10 54,722 tok/s peakVRAM 13.7GB +[19:23:39] step 2020/4768 loss 3.2152 lr 6.00e-04 gnorm 0.10 54,720 tok/s peakVRAM 13.7GB +[19:26:50] step 2040/4768 loss 3.2077 lr 6.00e-04 gnorm 0.11 54,721 tok/s peakVRAM 13.7GB +[19:30:02] step 2060/4768 loss 3.2078 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB +[19:33:14] step 2080/4768 loss 3.1967 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB +[19:36:25] step 2100/4768 loss 3.2051 lr 6.00e-04 gnorm 0.11 54,718 tok/s peakVRAM 13.7GB +[19:39:37] step 2120/4768 loss 3.1951 lr 6.00e-04 gnorm 0.11 54,717 tok/s peakVRAM 13.7GB +[19:42:49] step 2140/4768 loss 3.2076 lr 6.00e-04 gnorm 0.11 54,717 tok/s peakVRAM 13.7GB +[19:46:00] step 2160/4768 loss 3.1865 lr 6.00e-04 gnorm 0.10 54,714 tok/s peakVRAM 13.7GB +[19:49:12] step 2180/4768 loss 3.1881 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB +[19:52:24] step 2200/4768 loss 3.1892 lr 6.00e-04 gnorm 0.10 54,711 tok/s peakVRAM 13.7GB +[19:55:35] step 2220/4768 loss 3.1820 lr 6.00e-04 gnorm 0.10 54,709 tok/s peakVRAM 13.7GB +[19:58:47] step 2240/4768 loss 3.1787 lr 6.00e-04 gnorm 0.09 54,712 tok/s peakVRAM 13.7GB +[20:01:58] step 2260/4768 loss 3.1754 lr 6.00e-04 gnorm 0.10 54,714 tok/s peakVRAM 13.7GB +[20:05:10] step 2280/4768 loss 3.1711 lr 6.00e-04 gnorm 0.11 54,712 tok/s peakVRAM 13.7GB +[20:08:22] step 2300/4768 loss 3.1747 lr 6.00e-04 gnorm 0.09 54,716 tok/s peakVRAM 13.7GB +[20:11:33] step 2320/4768 loss 3.1717 lr 6.00e-04 gnorm 0.11 54,718 tok/s peakVRAM 13.7GB +[20:14:45] step 2340/4768 loss 3.1667 lr 6.00e-04 gnorm 0.11 54,715 tok/s peakVRAM 13.7GB +[20:17:57] step 2360/4768 loss 3.1573 lr 6.00e-04 gnorm 0.10 54,718 tok/s peakVRAM 13.7GB +[20:21:08] step 2380/4768 loss 3.1559 lr 6.00e-04 gnorm 0.10 54,714 tok/s peakVRAM 13.7GB +[20:24:20] step 2400/4768 loss 3.1566 lr 6.00e-04 gnorm 0.09 54,718 tok/s peakVRAM 13.7GB +[20:27:32] step 2420/4768 loss 3.1597 lr 6.00e-04 gnorm 0.10 54,720 tok/s peakVRAM 13.7GB +[20:30:43] step 2440/4768 loss 3.1548 lr 6.00e-04 gnorm 0.09 54,721 tok/s peakVRAM 13.7GB +[20:33:55] step 2460/4768 loss 3.1532 lr 6.00e-04 gnorm 0.11 54,719 tok/s peakVRAM 13.7GB +[20:37:06] step 2480/4768 loss 3.1504 lr 6.00e-04 gnorm 0.11 54,717 tok/s peakVRAM 13.7GB +[20:40:18] step 2500/4768 loss 3.1447 lr 6.00e-04 gnorm 0.10 54,716 tok/s peakVRAM 13.7GB +[20:43:30] step 2520/4768 loss 3.1466 lr 6.00e-04 gnorm 0.10 54,719 tok/s peakVRAM 13.7GB +[20:46:41] step 2540/4768 loss 3.1465 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB +[20:49:53] step 2560/4768 loss 3.1434 lr 6.00e-04 gnorm 0.11 54,719 tok/s peakVRAM 13.7GB +[20:53:05] step 2580/4768 loss 3.1385 lr 6.00e-04 gnorm 0.10 54,719 tok/s peakVRAM 13.7GB +[20:56:16] step 2600/4768 loss 3.1294 lr 6.00e-04 gnorm 0.10 54,716 tok/s peakVRAM 13.7GB +[20:59:28] step 2620/4768 loss 3.1375 lr 6.00e-04 gnorm 0.09 54,712 tok/s peakVRAM 13.7GB +[21:02:40] step 2640/4768 loss 3.1389 lr 6.00e-04 gnorm 0.10 54,714 tok/s peakVRAM 13.7GB +[21:05:51] step 2660/4768 loss 3.1372 lr 6.00e-04 gnorm 0.10 54,714 tok/s peakVRAM 13.7GB +[21:09:03] step 2680/4768 loss 3.1326 lr 6.00e-04 gnorm 0.10 54,716 tok/s peakVRAM 13.7GB +[21:12:14] step 2700/4768 loss 3.1352 lr 6.00e-04 gnorm 0.10 54,718 tok/s peakVRAM 13.7GB +[21:15:26] step 2720/4768 loss 3.1180 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB +[21:18:38] step 2740/4768 loss 3.1193 lr 6.00e-04 gnorm 0.10 54,723 tok/s peakVRAM 13.7GB +[21:21:49] step 2760/4768 loss 3.1220 lr 6.00e-04 gnorm 0.10 54,724 tok/s peakVRAM 13.7GB +[21:25:01] step 2780/4768 loss 3.1132 lr 6.00e-04 gnorm 0.10 54,722 tok/s peakVRAM 13.7GB +[21:28:13] step 2800/4768 loss 3.1081 lr 6.00e-04 gnorm 0.10 54,720 tok/s peakVRAM 13.7GB +[21:31:24] step 2820/4768 loss 3.1058 lr 6.00e-04 gnorm 0.10 54,719 tok/s peakVRAM 13.7GB +[21:34:36] step 2840/4768 loss 3.1134 lr 6.00e-04 gnorm 0.10 54,719 tok/s peakVRAM 13.7GB +[21:37:47] step 2860/4768 loss 3.1055 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB +[21:40:59] step 2880/4768 loss 3.1071 lr 6.00e-04 gnorm 0.09 54,714 tok/s peakVRAM 13.7GB +[21:44:11] step 2900/4768 loss 3.1088 lr 6.00e-04 gnorm 0.11 54,713 tok/s peakVRAM 13.7GB +[21:47:22] step 2920/4768 loss 3.1159 lr 6.00e-04 gnorm 0.10 54,718 tok/s peakVRAM 13.7GB +[21:50:34] step 2940/4768 loss 3.1065 lr 6.00e-04 gnorm 0.08 54,720 tok/s peakVRAM 13.7GB +[21:53:46] step 2960/4768 loss 3.1085 lr 6.00e-04 gnorm 0.10 54,720 tok/s peakVRAM 13.7GB +[21:56:57] step 2980/4768 loss 3.0954 lr 6.00e-04 gnorm 0.10 54,721 tok/s peakVRAM 13.7GB +[22:00:09] step 3000/4768 loss 3.0957 lr 6.00e-04 gnorm 0.09 54,721 tok/s peakVRAM 13.7GB +[22:03:21] step 3020/4768 loss 3.0920 lr 6.00e-04 gnorm 0.09 54,721 tok/s peakVRAM 13.7GB +[22:06:32] step 3040/4768 loss 3.0889 lr 6.00e-04 gnorm 0.09 54,714 tok/s peakVRAM 13.7GB +[22:09:44] step 3060/4768 loss 3.0923 lr 6.00e-04 gnorm 0.09 54,718 tok/s peakVRAM 13.7GB +[22:12:55] step 3080/4768 loss 3.0847 lr 6.00e-04 gnorm 0.09 54,720 tok/s peakVRAM 13.7GB +[22:16:07] step 3100/4768 loss 3.0786 lr 6.00e-04 gnorm 0.10 54,725 tok/s peakVRAM 13.7GB +[22:19:19] step 3120/4768 loss 3.0947 lr 6.00e-04 gnorm 0.10 54,721 tok/s peakVRAM 13.7GB +[22:22:30] step 3140/4768 loss 3.0798 lr 6.00e-04 gnorm 0.10 54,724 tok/s peakVRAM 13.7GB +[22:25:42] step 3160/4768 loss 3.0761 lr 6.00e-04 gnorm 0.09 54,722 tok/s peakVRAM 13.7GB +[22:28:54] step 3180/4768 loss 3.0823 lr 6.00e-04 gnorm 0.09 54,718 tok/s peakVRAM 13.7GB +[22:32:05] step 3200/4768 loss 3.0832 lr 6.00e-04 gnorm 0.09 54,720 tok/s peakVRAM 13.7GB +[22:35:17] step 3220/4768 loss 3.0926 lr 6.00e-04 gnorm 0.10 54,722 tok/s peakVRAM 13.7GB +[22:38:28] step 3240/4768 loss 3.0746 lr 6.00e-04 gnorm 0.09 54,719 tok/s peakVRAM 13.7GB +[22:41:40] step 3260/4768 loss 3.0712 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB +[22:44:52] step 3280/4768 loss 3.0670 lr 6.00e-04 gnorm 0.09 54,719 tok/s peakVRAM 13.7GB +[22:48:03] step 3300/4768 loss 3.0780 lr 6.00e-04 gnorm 0.09 54,712 tok/s peakVRAM 13.7GB +[22:51:15] step 3320/4768 loss 3.0718 lr 6.00e-04 gnorm 0.08 54,713 tok/s peakVRAM 13.7GB +[22:54:27] step 3340/4768 loss 3.0722 lr 6.00e-04 gnorm 0.09 54,716 tok/s peakVRAM 13.7GB +[22:57:38] step 3360/4768 loss 3.0630 lr 6.00e-04 gnorm 0.10 54,718 tok/s peakVRAM 13.7GB +[23:00:50] step 3380/4768 loss 3.0728 lr 6.00e-04 gnorm 0.10 54,715 tok/s peakVRAM 13.7GB +[23:04:02] step 3400/4768 loss 3.0580 lr 6.00e-04 gnorm 0.09 54,719 tok/s peakVRAM 13.7GB +[23:07:13] step 3420/4768 loss 3.0660 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB +[23:10:25] step 3440/4768 loss 3.0576 lr 6.00e-04 gnorm 0.10 54,716 tok/s peakVRAM 13.7GB +[23:13:36] step 3460/4768 loss 3.0667 lr 6.00e-04 gnorm 0.09 54,715 tok/s peakVRAM 13.7GB +[23:16:48] step 3480/4768 loss 3.0667 lr 6.00e-04 gnorm 0.09 54,716 tok/s peakVRAM 13.7GB +[23:20:00] step 3500/4768 loss 3.0526 lr 6.00e-04 gnorm 0.10 54,714 tok/s peakVRAM 13.7GB +[23:23:11] step 3520/4768 loss 3.0552 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB +[23:26:23] step 3540/4768 loss 3.0578 lr 6.00e-04 gnorm 0.09 54,716 tok/s peakVRAM 13.7GB +[23:29:35] step 3560/4768 loss 3.0616 lr 6.00e-04 gnorm 0.09 54,717 tok/s peakVRAM 13.7GB +[23:32:46] step 3580/4768 loss 3.0558 lr 6.00e-04 gnorm 0.08 54,722 tok/s peakVRAM 13.7GB +[23:35:58] step 3600/4768 loss 3.0576 lr 6.00e-04 gnorm 0.10 54,721 tok/s peakVRAM 13.7GB +[23:39:10] step 3620/4768 loss 3.0497 lr 6.00e-04 gnorm 0.10 54,719 tok/s peakVRAM 13.7GB +[23:42:21] step 3640/4768 loss 3.0574 lr 6.00e-04 gnorm 0.09 54,722 tok/s peakVRAM 13.7GB +[23:45:33] step 3660/4768 loss 3.0462 lr 6.00e-04 gnorm 0.09 54,720 tok/s peakVRAM 13.7GB +[23:48:44] step 3680/4768 loss 3.0531 lr 6.00e-04 gnorm 0.09 54,724 tok/s peakVRAM 13.7GB +[23:51:56] step 3700/4768 loss 3.0503 lr 6.00e-04 gnorm 0.09 54,726 tok/s peakVRAM 13.7GB +[23:55:08] step 3720/4768 loss 3.0435 lr 6.00e-04 gnorm 0.08 54,724 tok/s peakVRAM 13.7GB +[23:58:19] step 3740/4768 loss 3.0400 lr 6.00e-04 gnorm 0.08 54,727 tok/s peakVRAM 13.7GB +[00:01:31] step 3760/4768 loss 3.0384 lr 6.00e-04 gnorm 0.09 54,725 tok/s peakVRAM 13.7GB +[00:04:42] step 3780/4768 loss 3.0390 lr 6.00e-04 gnorm 0.10 54,721 tok/s peakVRAM 13.7GB +[00:07:54] step 3800/4768 loss 3.0400 lr 6.00e-04 gnorm 0.09 54,718 tok/s peakVRAM 13.7GB +[00:11:06] step 3820/4768 loss 3.0331 lr 5.61e-04 gnorm 0.09 54,718 tok/s peakVRAM 13.7GB +[00:14:17] step 3840/4768 loss 3.0316 lr 5.13e-04 gnorm 0.08 54,717 tok/s peakVRAM 13.7GB +[00:17:29] step 3860/4768 loss 3.0245 lr 4.83e-04 gnorm 0.07 54,718 tok/s peakVRAM 13.7GB +[00:20:41] step 3880/4768 loss 3.0217 lr 4.59e-04 gnorm 0.08 54,721 tok/s peakVRAM 13.7GB +[00:23:52] step 3900/4768 loss 3.0150 lr 4.39e-04 gnorm 0.08 54,716 tok/s peakVRAM 13.7GB +[00:27:04] step 3920/4768 loss 3.0093 lr 4.21e-04 gnorm 0.07 54,715 tok/s peakVRAM 13.7GB +[00:30:16] step 3940/4768 loss 3.0019 lr 4.05e-04 gnorm 0.07 54,715 tok/s peakVRAM 13.7GB +[00:33:27] step 3960/4768 loss 3.0024 lr 3.89e-04 gnorm 0.08 54,716 tok/s peakVRAM 13.7GB +[00:36:39] step 3980/4768 loss 2.9945 lr 3.75e-04 gnorm 0.07 54,721 tok/s peakVRAM 13.7GB +[00:39:50] step 4000/4768 loss 2.9933 lr 3.62e-04 gnorm 0.07 54,722 tok/s peakVRAM 13.7GB +[00:43:02] step 4020/4768 loss 2.9906 lr 3.50e-04 gnorm 0.07 54,713 tok/s peakVRAM 13.7GB +[00:46:14] step 4040/4768 loss 2.9812 lr 3.38e-04 gnorm 0.07 54,714 tok/s peakVRAM 13.7GB +[00:49:25] step 4060/4768 loss 2.9809 lr 3.26e-04 gnorm 0.07 54,719 tok/s peakVRAM 13.7GB +[00:52:37] step 4080/4768 loss 2.9725 lr 3.15e-04 gnorm 0.07 54,715 tok/s peakVRAM 13.7GB +[00:55:49] step 4100/4768 loss 2.9776 lr 3.05e-04 gnorm 0.07 54,716 tok/s peakVRAM 13.7GB +[00:59:00] step 4120/4768 loss 2.9664 lr 2.95e-04 gnorm 0.07 54,720 tok/s peakVRAM 13.7GB +[01:02:12] step 4140/4768 loss 2.9586 lr 2.85e-04 gnorm 0.07 54,716 tok/s peakVRAM 13.7GB +[01:05:24] step 4160/4768 loss 2.9768 lr 2.75e-04 gnorm 0.07 54,710 tok/s peakVRAM 13.7GB +[01:08:35] step 4180/4768 loss 2.9694 lr 2.66e-04 gnorm 0.07 54,712 tok/s peakVRAM 13.7GB +[01:11:47] step 4200/4768 loss 2.9631 lr 2.57e-04 gnorm 0.07 54,715 tok/s peakVRAM 13.7GB +[01:14:58] step 4220/4768 loss 2.9561 lr 2.48e-04 gnorm 0.07 54,716 tok/s peakVRAM 13.7GB +[01:18:10] step 4240/4768 loss 2.9624 lr 2.40e-04 gnorm 0.07 54,718 tok/s peakVRAM 13.7GB +[01:21:22] step 4260/4768 loss 2.9546 lr 2.31e-04 gnorm 0.07 54,720 tok/s peakVRAM 13.7GB +[01:24:33] step 4280/4768 loss 2.9540 lr 2.23e-04 gnorm 0.07 54,725 tok/s peakVRAM 13.7GB +[01:27:45] step 4300/4768 loss 2.9518 lr 2.15e-04 gnorm 0.07 54,721 tok/s peakVRAM 13.7GB +[01:30:57] step 4320/4768 loss 2.9470 lr 2.07e-04 gnorm 0.07 54,718 tok/s peakVRAM 13.7GB +[01:34:08] step 4340/4768 loss 2.9457 lr 1.99e-04 gnorm 0.07 54,717 tok/s peakVRAM 13.7GB +[01:37:20] step 4360/4768 loss 2.9396 lr 1.92e-04 gnorm 0.07 54,717 tok/s peakVRAM 13.7GB +[01:40:32] step 4380/4768 loss 2.9465 lr 1.84e-04 gnorm 0.07 54,713 tok/s peakVRAM 13.7GB +[01:43:43] step 4400/4768 loss 2.9439 lr 1.77e-04 gnorm 0.07 54,711 tok/s peakVRAM 13.7GB +[01:46:55] step 4420/4768 loss 2.9371 lr 1.70e-04 gnorm 0.07 54,714 tok/s peakVRAM 13.7GB +[01:50:06] step 4440/4768 loss 2.9435 lr 1.63e-04 gnorm 0.07 54,713 tok/s peakVRAM 13.7GB +[01:53:18] step 4460/4768 loss 2.9390 lr 1.56e-04 gnorm 0.07 54,716 tok/s peakVRAM 13.7GB +[01:56:30] step 4480/4768 loss 2.9341 lr 1.49e-04 gnorm 0.07 54,719 tok/s peakVRAM 13.7GB +[01:59:41] step 4500/4768 loss 2.9313 lr 1.42e-04 gnorm 0.07 54,720 tok/s peakVRAM 13.7GB +[02:02:53] step 4520/4768 loss 2.9332 lr 1.36e-04 gnorm 0.07 54,717 tok/s peakVRAM 13.7GB +[02:06:05] step 4540/4768 loss 2.9246 lr 1.29e-04 gnorm 0.07 54,717 tok/s peakVRAM 13.7GB +[02:09:16] step 4560/4768 loss 2.9280 lr 1.23e-04 gnorm 0.07 54,712 tok/s peakVRAM 13.7GB +[02:12:28] step 4580/4768 loss 2.9309 lr 1.16e-04 gnorm 0.07 54,710 tok/s peakVRAM 13.7GB +[02:15:40] step 4600/4768 loss 2.9299 lr 1.10e-04 gnorm 0.07 54,708 tok/s peakVRAM 13.7GB +[02:18:51] step 4620/4768 loss 2.9163 lr 1.04e-04 gnorm 0.07 54,711 tok/s peakVRAM 13.7GB +[02:22:03] step 4640/4768 loss 2.9256 lr 9.78e-05 gnorm 0.07 54,714 tok/s peakVRAM 13.7GB +[02:25:15] step 4660/4768 loss 2.9184 lr 9.18e-05 gnorm 0.07 54,713 tok/s peakVRAM 13.7GB +[02:28:26] step 4680/4768 loss 2.9159 lr 8.58e-05 gnorm 0.07 54,711 tok/s peakVRAM 13.7GB +[02:31:38] step 4700/4768 loss 2.9121 lr 7.99e-05 gnorm 0.07 54,712 tok/s peakVRAM 13.7GB +[02:34:50] step 4720/4768 loss 2.9131 lr 7.41e-05 gnorm 0.07 54,715 tok/s peakVRAM 13.7GB +[02:38:01] step 4740/4768 loss 2.9110 lr 6.83e-05 gnorm 0.07 54,718 tok/s peakVRAM 13.7GB +[02:41:13] step 4760/4768 loss 2.9202 lr 6.26e-05 gnorm 0.07 54,720 tok/s peakVRAM 13.7GB +[02:42:31] ckpt @ 4768 +[02:42:31] DONE diff --git a/R2/run/train.out b/R2/run/train.out new file mode 100644 index 0000000000000000000000000000000000000000..260766e962072532469d66075125d8b6454dc8f2 --- /dev/null +++ b/R2/run/train.out @@ -0,0 +1,248 @@ +[13:59:56] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[13:59:56] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[13:59:58] model GPT-ref: 183.6M param (L16 d896 h14) +[13:59:59] optimizer=muon muon_lr=0.02 (mult=33.3x) +[13:59:59] r11 pamiec: act_ckpt=True loss_chunk=1024 +[13:59:59] r13: mixer_layout=AAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[13:59:59] torch.compile enabled +[13:59:59] START gpt-ref: steps=4768 (od 0) lr=0.0006->6e-05 +[14:04:20] step 20/4768 loss 10.3979 lr 2.40e-05 gnorm 2.60 40,239 tok/s peakVRAM 13.7GB +[14:07:31] step 40/4768 loss 9.5831 lr 4.80e-05 gnorm 1.42 54,808 tok/s peakVRAM 13.7GB +[14:10:43] step 60/4768 loss 8.7021 lr 7.20e-05 gnorm 0.76 54,817 tok/s peakVRAM 13.7GB +[14:13:54] step 80/4768 loss 8.0482 lr 9.60e-05 gnorm 1.57 54,813 tok/s peakVRAM 13.7GB +[14:17:05] step 100/4768 loss 7.6036 lr 1.20e-04 gnorm 0.99 54,787 tok/s peakVRAM 13.7GB +[14:20:17] step 120/4768 loss 7.1345 lr 1.44e-04 gnorm 0.74 54,772 tok/s peakVRAM 13.7GB +[14:23:28] step 140/4768 loss 6.7198 lr 1.68e-04 gnorm 0.81 54,761 tok/s peakVRAM 13.7GB +[14:26:40] step 160/4768 loss 6.3578 lr 1.92e-04 gnorm 0.82 54,756 tok/s peakVRAM 13.7GB +[14:29:51] step 180/4768 loss 6.0886 lr 2.16e-04 gnorm 0.72 54,754 tok/s peakVRAM 13.7GB +[14:33:03] step 200/4768 loss 5.8673 lr 2.40e-04 gnorm 0.70 54,750 tok/s peakVRAM 13.7GB +[14:36:14] step 220/4768 loss 5.6795 lr 2.64e-04 gnorm 0.63 54,744 tok/s peakVRAM 13.7GB +[14:39:26] step 240/4768 loss 5.5068 lr 2.88e-04 gnorm 0.73 54,744 tok/s peakVRAM 13.7GB +[14:42:37] step 260/4768 loss 5.3420 lr 3.12e-04 gnorm 0.89 54,745 tok/s peakVRAM 13.7GB +[14:45:49] step 280/4768 loss 5.1838 lr 3.36e-04 gnorm 0.74 54,746 tok/s peakVRAM 13.7GB +[14:49:00] step 300/4768 loss 5.0376 lr 3.60e-04 gnorm 0.77 54,747 tok/s peakVRAM 13.7GB +[14:52:12] step 320/4768 loss 4.9032 lr 3.84e-04 gnorm 0.87 54,747 tok/s peakVRAM 13.7GB +[14:55:24] step 340/4768 loss 4.7531 lr 4.08e-04 gnorm 0.75 54,744 tok/s peakVRAM 13.7GB +[14:58:35] step 360/4768 loss 4.6273 lr 4.32e-04 gnorm 0.59 54,745 tok/s peakVRAM 13.7GB +[15:01:47] step 380/4768 loss 4.5078 lr 4.56e-04 gnorm 0.56 54,744 tok/s peakVRAM 13.7GB +[15:04:58] step 400/4768 loss 4.4062 lr 4.80e-04 gnorm 0.45 54,745 tok/s peakVRAM 13.7GB +[15:08:10] step 420/4768 loss 4.3287 lr 5.04e-04 gnorm 0.46 54,747 tok/s peakVRAM 13.7GB +[15:11:21] step 440/4768 loss 4.2505 lr 5.28e-04 gnorm 0.40 54,744 tok/s peakVRAM 13.7GB +[15:14:33] step 460/4768 loss 4.1806 lr 5.52e-04 gnorm 0.39 54,744 tok/s peakVRAM 13.7GB +[15:17:44] step 480/4768 loss 4.1334 lr 5.76e-04 gnorm 0.35 54,742 tok/s peakVRAM 13.7GB +[15:20:56] step 500/4768 loss 4.0774 lr 6.00e-04 gnorm 0.31 54,736 tok/s peakVRAM 13.7GB +[15:24:07] step 520/4768 loss 4.0397 lr 6.00e-04 gnorm 0.27 54,735 tok/s peakVRAM 13.7GB +[15:27:19] step 540/4768 loss 3.9894 lr 6.00e-04 gnorm 0.22 54,736 tok/s peakVRAM 13.7GB +[15:30:31] step 560/4768 loss 3.9379 lr 6.00e-04 gnorm 0.23 54,734 tok/s peakVRAM 13.7GB +[15:33:42] step 580/4768 loss 3.8968 lr 6.00e-04 gnorm 0.23 54,732 tok/s peakVRAM 13.7GB +[15:36:54] step 600/4768 loss 3.8534 lr 6.00e-04 gnorm 0.21 54,728 tok/s peakVRAM 13.7GB +[15:40:05] step 620/4768 loss 3.8302 lr 6.00e-04 gnorm 0.21 54,730 tok/s peakVRAM 13.7GB +[15:43:17] step 640/4768 loss 3.8048 lr 6.00e-04 gnorm 0.19 54,735 tok/s peakVRAM 13.7GB +[15:46:29] step 660/4768 loss 3.7741 lr 6.00e-04 gnorm 0.17 54,729 tok/s peakVRAM 13.7GB +[15:49:40] step 680/4768 loss 3.7513 lr 6.00e-04 gnorm 0.18 54,735 tok/s peakVRAM 13.7GB +[15:52:52] step 700/4768 loss 3.7199 lr 6.00e-04 gnorm 0.18 54,736 tok/s peakVRAM 13.7GB +[15:56:03] step 720/4768 loss 3.6895 lr 6.00e-04 gnorm 0.17 54,733 tok/s peakVRAM 13.7GB +[15:59:15] step 740/4768 loss 3.6858 lr 6.00e-04 gnorm 0.19 54,732 tok/s peakVRAM 13.7GB +[16:02:26] step 760/4768 loss 3.6607 lr 6.00e-04 gnorm 0.16 54,731 tok/s peakVRAM 13.7GB +[16:05:38] step 780/4768 loss 3.6347 lr 6.00e-04 gnorm 0.16 54,731 tok/s peakVRAM 13.7GB +[16:08:50] step 800/4768 loss 3.6206 lr 6.00e-04 gnorm 0.14 54,732 tok/s peakVRAM 13.7GB +[16:12:01] step 820/4768 loss 3.6040 lr 6.00e-04 gnorm 0.14 54,730 tok/s peakVRAM 13.7GB +[16:15:13] step 840/4768 loss 3.5898 lr 6.00e-04 gnorm 0.15 54,728 tok/s peakVRAM 13.7GB +[16:18:24] step 860/4768 loss 3.5694 lr 6.00e-04 gnorm 0.16 54,730 tok/s peakVRAM 13.7GB +[16:21:36] step 880/4768 loss 3.5635 lr 6.00e-04 gnorm 0.14 54,727 tok/s peakVRAM 13.7GB +[16:24:48] step 900/4768 loss 3.5456 lr 6.00e-04 gnorm 0.14 54,728 tok/s peakVRAM 13.7GB +[16:27:59] step 920/4768 loss 3.5379 lr 6.00e-04 gnorm 0.14 54,732 tok/s peakVRAM 13.7GB +[16:31:11] step 940/4768 loss 3.5332 lr 6.00e-04 gnorm 0.14 54,731 tok/s peakVRAM 13.7GB +[16:34:22] step 960/4768 loss 3.5076 lr 6.00e-04 gnorm 0.13 54,727 tok/s peakVRAM 13.7GB +[16:37:34] step 980/4768 loss 3.5049 lr 6.00e-04 gnorm 0.12 54,728 tok/s peakVRAM 13.7GB +[16:40:46] step 1000/4768 loss 3.4945 lr 6.00e-04 gnorm 0.14 54,723 tok/s peakVRAM 13.7GB +[16:43:57] step 1020/4768 loss 3.4891 lr 6.00e-04 gnorm 0.13 54,721 tok/s peakVRAM 13.7GB +[16:47:09] step 1040/4768 loss 3.4625 lr 6.00e-04 gnorm 0.13 54,718 tok/s peakVRAM 13.7GB +[16:50:20] step 1060/4768 loss 3.4738 lr 6.00e-04 gnorm 0.14 54,717 tok/s peakVRAM 13.7GB +[16:53:32] step 1080/4768 loss 3.4629 lr 6.00e-04 gnorm 0.12 54,721 tok/s peakVRAM 13.7GB +[16:56:44] step 1100/4768 loss 3.4473 lr 6.00e-04 gnorm 0.14 54,724 tok/s peakVRAM 13.7GB +[16:59:55] step 1120/4768 loss 3.4297 lr 6.00e-04 gnorm 0.12 54,724 tok/s peakVRAM 13.7GB +[17:03:07] step 1140/4768 loss 3.4228 lr 6.00e-04 gnorm 0.12 54,728 tok/s peakVRAM 13.7GB +[17:06:19] step 1160/4768 loss 3.4173 lr 6.00e-04 gnorm 0.12 54,724 tok/s peakVRAM 13.7GB +[17:09:30] step 1180/4768 loss 3.4195 lr 6.00e-04 gnorm 0.12 54,725 tok/s peakVRAM 13.7GB +[17:12:42] step 1200/4768 loss 3.4018 lr 6.00e-04 gnorm 0.13 54,724 tok/s peakVRAM 13.7GB +[17:15:53] step 1220/4768 loss 3.3993 lr 6.00e-04 gnorm 0.13 54,726 tok/s peakVRAM 13.7GB +[17:19:05] step 1240/4768 loss 3.3916 lr 6.00e-04 gnorm 0.12 54,728 tok/s peakVRAM 13.7GB +[17:22:17] step 1260/4768 loss 3.3782 lr 6.00e-04 gnorm 0.12 54,729 tok/s peakVRAM 13.7GB +[17:25:28] step 1280/4768 loss 3.3807 lr 6.00e-04 gnorm 0.12 54,722 tok/s peakVRAM 13.7GB +[17:28:40] step 1300/4768 loss 3.3735 lr 6.00e-04 gnorm 0.11 54,719 tok/s peakVRAM 13.7GB +[17:31:51] step 1320/4768 loss 3.3659 lr 6.00e-04 gnorm 0.11 54,718 tok/s peakVRAM 13.7GB +[17:35:03] step 1340/4768 loss 3.3664 lr 6.00e-04 gnorm 0.13 54,719 tok/s peakVRAM 13.7GB +[17:38:15] step 1360/4768 loss 3.3575 lr 6.00e-04 gnorm 0.13 54,716 tok/s peakVRAM 13.7GB +[17:41:26] step 1380/4768 loss 3.3491 lr 6.00e-04 gnorm 0.11 54,722 tok/s peakVRAM 13.7GB +[17:44:38] step 1400/4768 loss 3.3387 lr 6.00e-04 gnorm 0.12 54,724 tok/s peakVRAM 13.7GB +[17:47:50] step 1420/4768 loss 3.3266 lr 6.00e-04 gnorm 0.12 54,725 tok/s peakVRAM 13.7GB +[17:51:01] step 1440/4768 loss 3.3337 lr 6.00e-04 gnorm 0.11 54,722 tok/s peakVRAM 13.7GB +[17:54:13] step 1460/4768 loss 3.3312 lr 6.00e-04 gnorm 0.11 54,722 tok/s peakVRAM 13.7GB +[17:57:24] step 1480/4768 loss 3.3352 lr 6.00e-04 gnorm 0.11 54,719 tok/s peakVRAM 13.7GB +[18:00:36] step 1500/4768 loss 3.3080 lr 6.00e-04 gnorm 0.10 54,720 tok/s peakVRAM 13.7GB +[18:03:48] step 1520/4768 loss 3.3154 lr 6.00e-04 gnorm 0.12 54,719 tok/s peakVRAM 13.7GB +[18:06:59] step 1540/4768 loss 3.2992 lr 6.00e-04 gnorm 0.11 54,721 tok/s peakVRAM 13.7GB +[18:10:11] step 1560/4768 loss 3.3079 lr 6.00e-04 gnorm 0.13 54,718 tok/s peakVRAM 13.7GB +[18:13:23] step 1580/4768 loss 3.2967 lr 6.00e-04 gnorm 0.11 54,719 tok/s peakVRAM 13.7GB +[18:16:34] step 1600/4768 loss 3.2853 lr 6.00e-04 gnorm 0.11 54,716 tok/s peakVRAM 13.7GB +[18:19:46] step 1620/4768 loss 3.2858 lr 6.00e-04 gnorm 0.12 54,719 tok/s peakVRAM 13.7GB +[18:22:57] step 1640/4768 loss 3.2827 lr 6.00e-04 gnorm 0.12 54,718 tok/s peakVRAM 13.7GB +[18:26:09] step 1660/4768 loss 3.2839 lr 6.00e-04 gnorm 0.11 54,723 tok/s peakVRAM 13.7GB +[18:29:21] step 1680/4768 loss 3.2733 lr 6.00e-04 gnorm 0.11 54,717 tok/s peakVRAM 13.7GB +[18:32:32] step 1700/4768 loss 3.2658 lr 6.00e-04 gnorm 0.12 54,718 tok/s peakVRAM 13.7GB +[18:35:44] step 1720/4768 loss 3.2679 lr 6.00e-04 gnorm 0.12 54,712 tok/s peakVRAM 13.7GB +[18:38:56] step 1740/4768 loss 3.2618 lr 6.00e-04 gnorm 0.10 54,710 tok/s peakVRAM 13.7GB +[18:42:07] step 1760/4768 loss 3.2606 lr 6.00e-04 gnorm 0.11 54,712 tok/s peakVRAM 13.7GB +[18:45:19] step 1780/4768 loss 3.2480 lr 6.00e-04 gnorm 0.11 54,715 tok/s peakVRAM 13.7GB +[18:48:31] step 1800/4768 loss 3.2525 lr 6.00e-04 gnorm 0.12 54,716 tok/s peakVRAM 13.7GB +[18:51:42] step 1820/4768 loss 3.2466 lr 6.00e-04 gnorm 0.11 54,717 tok/s peakVRAM 13.7GB +[18:54:54] step 1840/4768 loss 3.2411 lr 6.00e-04 gnorm 0.10 54,712 tok/s peakVRAM 13.7GB +[18:58:06] step 1860/4768 loss 3.2404 lr 6.00e-04 gnorm 0.12 54,710 tok/s peakVRAM 13.7GB +[19:01:17] step 1880/4768 loss 3.2358 lr 6.00e-04 gnorm 0.11 54,707 tok/s peakVRAM 13.7GB +[19:04:29] step 1900/4768 loss 3.2309 lr 6.00e-04 gnorm 0.10 54,706 tok/s peakVRAM 13.7GB +[19:07:41] step 1920/4768 loss 3.2304 lr 6.00e-04 gnorm 0.10 54,706 tok/s peakVRAM 13.7GB +[19:10:52] step 1940/4768 loss 3.2230 lr 6.00e-04 gnorm 0.12 54,710 tok/s peakVRAM 13.7GB +[19:14:04] step 1960/4768 loss 3.2270 lr 6.00e-04 gnorm 0.11 54,711 tok/s peakVRAM 13.7GB +[19:17:16] step 1980/4768 loss 3.2220 lr 6.00e-04 gnorm 0.09 54,714 tok/s peakVRAM 13.7GB +[19:20:27] step 2000/4768 loss 3.2153 lr 6.00e-04 gnorm 0.10 54,722 tok/s peakVRAM 13.7GB +[19:23:39] step 2020/4768 loss 3.2152 lr 6.00e-04 gnorm 0.10 54,720 tok/s peakVRAM 13.7GB +[19:26:50] step 2040/4768 loss 3.2077 lr 6.00e-04 gnorm 0.11 54,721 tok/s peakVRAM 13.7GB +[19:30:02] step 2060/4768 loss 3.2078 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB +[19:33:14] step 2080/4768 loss 3.1967 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB +[19:36:25] step 2100/4768 loss 3.2051 lr 6.00e-04 gnorm 0.11 54,718 tok/s peakVRAM 13.7GB +[19:39:37] step 2120/4768 loss 3.1951 lr 6.00e-04 gnorm 0.11 54,717 tok/s peakVRAM 13.7GB +[19:42:49] step 2140/4768 loss 3.2076 lr 6.00e-04 gnorm 0.11 54,717 tok/s peakVRAM 13.7GB +[19:46:00] step 2160/4768 loss 3.1865 lr 6.00e-04 gnorm 0.10 54,714 tok/s peakVRAM 13.7GB +[19:49:12] step 2180/4768 loss 3.1881 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB +[19:52:24] step 2200/4768 loss 3.1892 lr 6.00e-04 gnorm 0.10 54,711 tok/s peakVRAM 13.7GB +[19:55:35] step 2220/4768 loss 3.1820 lr 6.00e-04 gnorm 0.10 54,709 tok/s peakVRAM 13.7GB +[19:58:47] step 2240/4768 loss 3.1787 lr 6.00e-04 gnorm 0.09 54,712 tok/s peakVRAM 13.7GB +[20:01:58] step 2260/4768 loss 3.1754 lr 6.00e-04 gnorm 0.10 54,714 tok/s peakVRAM 13.7GB +[20:05:10] step 2280/4768 loss 3.1711 lr 6.00e-04 gnorm 0.11 54,712 tok/s peakVRAM 13.7GB +[20:08:22] step 2300/4768 loss 3.1747 lr 6.00e-04 gnorm 0.09 54,716 tok/s peakVRAM 13.7GB +[20:11:33] step 2320/4768 loss 3.1717 lr 6.00e-04 gnorm 0.11 54,718 tok/s peakVRAM 13.7GB +[20:14:45] step 2340/4768 loss 3.1667 lr 6.00e-04 gnorm 0.11 54,715 tok/s peakVRAM 13.7GB +[20:17:57] step 2360/4768 loss 3.1573 lr 6.00e-04 gnorm 0.10 54,718 tok/s peakVRAM 13.7GB +[20:21:08] step 2380/4768 loss 3.1559 lr 6.00e-04 gnorm 0.10 54,714 tok/s peakVRAM 13.7GB +[20:24:20] step 2400/4768 loss 3.1566 lr 6.00e-04 gnorm 0.09 54,718 tok/s peakVRAM 13.7GB +[20:27:32] step 2420/4768 loss 3.1597 lr 6.00e-04 gnorm 0.10 54,720 tok/s peakVRAM 13.7GB +[20:30:43] step 2440/4768 loss 3.1548 lr 6.00e-04 gnorm 0.09 54,721 tok/s peakVRAM 13.7GB +[20:33:55] step 2460/4768 loss 3.1532 lr 6.00e-04 gnorm 0.11 54,719 tok/s peakVRAM 13.7GB +[20:37:06] step 2480/4768 loss 3.1504 lr 6.00e-04 gnorm 0.11 54,717 tok/s peakVRAM 13.7GB +[20:40:18] step 2500/4768 loss 3.1447 lr 6.00e-04 gnorm 0.10 54,716 tok/s peakVRAM 13.7GB +[20:43:30] step 2520/4768 loss 3.1466 lr 6.00e-04 gnorm 0.10 54,719 tok/s peakVRAM 13.7GB +[20:46:41] step 2540/4768 loss 3.1465 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB +[20:49:53] step 2560/4768 loss 3.1434 lr 6.00e-04 gnorm 0.11 54,719 tok/s peakVRAM 13.7GB +[20:53:05] step 2580/4768 loss 3.1385 lr 6.00e-04 gnorm 0.10 54,719 tok/s peakVRAM 13.7GB +[20:56:16] step 2600/4768 loss 3.1294 lr 6.00e-04 gnorm 0.10 54,716 tok/s peakVRAM 13.7GB +[20:59:28] step 2620/4768 loss 3.1375 lr 6.00e-04 gnorm 0.09 54,712 tok/s peakVRAM 13.7GB +[21:02:40] step 2640/4768 loss 3.1389 lr 6.00e-04 gnorm 0.10 54,714 tok/s peakVRAM 13.7GB +[21:05:51] step 2660/4768 loss 3.1372 lr 6.00e-04 gnorm 0.10 54,714 tok/s peakVRAM 13.7GB +[21:09:03] step 2680/4768 loss 3.1326 lr 6.00e-04 gnorm 0.10 54,716 tok/s peakVRAM 13.7GB +[21:12:14] step 2700/4768 loss 3.1352 lr 6.00e-04 gnorm 0.10 54,718 tok/s peakVRAM 13.7GB +[21:15:26] step 2720/4768 loss 3.1180 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB +[21:18:38] step 2740/4768 loss 3.1193 lr 6.00e-04 gnorm 0.10 54,723 tok/s peakVRAM 13.7GB +[21:21:49] step 2760/4768 loss 3.1220 lr 6.00e-04 gnorm 0.10 54,724 tok/s peakVRAM 13.7GB +[21:25:01] step 2780/4768 loss 3.1132 lr 6.00e-04 gnorm 0.10 54,722 tok/s peakVRAM 13.7GB +[21:28:13] step 2800/4768 loss 3.1081 lr 6.00e-04 gnorm 0.10 54,720 tok/s peakVRAM 13.7GB +[21:31:24] step 2820/4768 loss 3.1058 lr 6.00e-04 gnorm 0.10 54,719 tok/s peakVRAM 13.7GB +[21:34:36] step 2840/4768 loss 3.1134 lr 6.00e-04 gnorm 0.10 54,719 tok/s peakVRAM 13.7GB +[21:37:47] step 2860/4768 loss 3.1055 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB +[21:40:59] step 2880/4768 loss 3.1071 lr 6.00e-04 gnorm 0.09 54,714 tok/s peakVRAM 13.7GB +[21:44:11] step 2900/4768 loss 3.1088 lr 6.00e-04 gnorm 0.11 54,713 tok/s peakVRAM 13.7GB +[21:47:22] step 2920/4768 loss 3.1159 lr 6.00e-04 gnorm 0.10 54,718 tok/s peakVRAM 13.7GB +[21:50:34] step 2940/4768 loss 3.1065 lr 6.00e-04 gnorm 0.08 54,720 tok/s peakVRAM 13.7GB +[21:53:46] step 2960/4768 loss 3.1085 lr 6.00e-04 gnorm 0.10 54,720 tok/s peakVRAM 13.7GB +[21:56:57] step 2980/4768 loss 3.0954 lr 6.00e-04 gnorm 0.10 54,721 tok/s peakVRAM 13.7GB +[22:00:09] step 3000/4768 loss 3.0957 lr 6.00e-04 gnorm 0.09 54,721 tok/s peakVRAM 13.7GB +[22:03:21] step 3020/4768 loss 3.0920 lr 6.00e-04 gnorm 0.09 54,721 tok/s peakVRAM 13.7GB +[22:06:32] step 3040/4768 loss 3.0889 lr 6.00e-04 gnorm 0.09 54,714 tok/s peakVRAM 13.7GB +[22:09:44] step 3060/4768 loss 3.0923 lr 6.00e-04 gnorm 0.09 54,718 tok/s peakVRAM 13.7GB +[22:12:55] step 3080/4768 loss 3.0847 lr 6.00e-04 gnorm 0.09 54,720 tok/s peakVRAM 13.7GB +[22:16:07] step 3100/4768 loss 3.0786 lr 6.00e-04 gnorm 0.10 54,725 tok/s peakVRAM 13.7GB +[22:19:19] step 3120/4768 loss 3.0947 lr 6.00e-04 gnorm 0.10 54,721 tok/s peakVRAM 13.7GB +[22:22:30] step 3140/4768 loss 3.0798 lr 6.00e-04 gnorm 0.10 54,724 tok/s peakVRAM 13.7GB +[22:25:42] step 3160/4768 loss 3.0761 lr 6.00e-04 gnorm 0.09 54,722 tok/s peakVRAM 13.7GB +[22:28:54] step 3180/4768 loss 3.0823 lr 6.00e-04 gnorm 0.09 54,718 tok/s peakVRAM 13.7GB +[22:32:05] step 3200/4768 loss 3.0832 lr 6.00e-04 gnorm 0.09 54,720 tok/s peakVRAM 13.7GB +[22:35:17] step 3220/4768 loss 3.0926 lr 6.00e-04 gnorm 0.10 54,722 tok/s peakVRAM 13.7GB +[22:38:28] step 3240/4768 loss 3.0746 lr 6.00e-04 gnorm 0.09 54,719 tok/s peakVRAM 13.7GB +[22:41:40] step 3260/4768 loss 3.0712 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB +[22:44:52] step 3280/4768 loss 3.0670 lr 6.00e-04 gnorm 0.09 54,719 tok/s peakVRAM 13.7GB +[22:48:03] step 3300/4768 loss 3.0780 lr 6.00e-04 gnorm 0.09 54,712 tok/s peakVRAM 13.7GB +[22:51:15] step 3320/4768 loss 3.0718 lr 6.00e-04 gnorm 0.08 54,713 tok/s peakVRAM 13.7GB +[22:54:27] step 3340/4768 loss 3.0722 lr 6.00e-04 gnorm 0.09 54,716 tok/s peakVRAM 13.7GB +[22:57:38] step 3360/4768 loss 3.0630 lr 6.00e-04 gnorm 0.10 54,718 tok/s peakVRAM 13.7GB +[23:00:50] step 3380/4768 loss 3.0728 lr 6.00e-04 gnorm 0.10 54,715 tok/s peakVRAM 13.7GB +[23:04:02] step 3400/4768 loss 3.0580 lr 6.00e-04 gnorm 0.09 54,719 tok/s peakVRAM 13.7GB +[23:07:13] step 3420/4768 loss 3.0660 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB +[23:10:25] step 3440/4768 loss 3.0576 lr 6.00e-04 gnorm 0.10 54,716 tok/s peakVRAM 13.7GB +[23:13:36] step 3460/4768 loss 3.0667 lr 6.00e-04 gnorm 0.09 54,715 tok/s peakVRAM 13.7GB +[23:16:48] step 3480/4768 loss 3.0667 lr 6.00e-04 gnorm 0.09 54,716 tok/s peakVRAM 13.7GB +[23:20:00] step 3500/4768 loss 3.0526 lr 6.00e-04 gnorm 0.10 54,714 tok/s peakVRAM 13.7GB +[23:23:11] step 3520/4768 loss 3.0552 lr 6.00e-04 gnorm 0.10 54,717 tok/s peakVRAM 13.7GB +[23:26:23] step 3540/4768 loss 3.0578 lr 6.00e-04 gnorm 0.09 54,716 tok/s peakVRAM 13.7GB +[23:29:35] step 3560/4768 loss 3.0616 lr 6.00e-04 gnorm 0.09 54,717 tok/s peakVRAM 13.7GB +[23:32:46] step 3580/4768 loss 3.0558 lr 6.00e-04 gnorm 0.08 54,722 tok/s peakVRAM 13.7GB +[23:35:58] step 3600/4768 loss 3.0576 lr 6.00e-04 gnorm 0.10 54,721 tok/s peakVRAM 13.7GB +[23:39:10] step 3620/4768 loss 3.0497 lr 6.00e-04 gnorm 0.10 54,719 tok/s peakVRAM 13.7GB +[23:42:21] step 3640/4768 loss 3.0574 lr 6.00e-04 gnorm 0.09 54,722 tok/s peakVRAM 13.7GB +[23:45:33] step 3660/4768 loss 3.0462 lr 6.00e-04 gnorm 0.09 54,720 tok/s peakVRAM 13.7GB +[23:48:44] step 3680/4768 loss 3.0531 lr 6.00e-04 gnorm 0.09 54,724 tok/s peakVRAM 13.7GB +[23:51:56] step 3700/4768 loss 3.0503 lr 6.00e-04 gnorm 0.09 54,726 tok/s peakVRAM 13.7GB +[23:55:08] step 3720/4768 loss 3.0435 lr 6.00e-04 gnorm 0.08 54,724 tok/s peakVRAM 13.7GB +[23:58:19] step 3740/4768 loss 3.0400 lr 6.00e-04 gnorm 0.08 54,727 tok/s peakVRAM 13.7GB +[00:01:31] step 3760/4768 loss 3.0384 lr 6.00e-04 gnorm 0.09 54,725 tok/s peakVRAM 13.7GB +[00:04:42] step 3780/4768 loss 3.0390 lr 6.00e-04 gnorm 0.10 54,721 tok/s peakVRAM 13.7GB +[00:07:54] step 3800/4768 loss 3.0400 lr 6.00e-04 gnorm 0.09 54,718 tok/s peakVRAM 13.7GB +[00:11:06] step 3820/4768 loss 3.0331 lr 5.61e-04 gnorm 0.09 54,718 tok/s peakVRAM 13.7GB +[00:14:17] step 3840/4768 loss 3.0316 lr 5.13e-04 gnorm 0.08 54,717 tok/s peakVRAM 13.7GB +[00:17:29] step 3860/4768 loss 3.0245 lr 4.83e-04 gnorm 0.07 54,718 tok/s peakVRAM 13.7GB +[00:20:41] step 3880/4768 loss 3.0217 lr 4.59e-04 gnorm 0.08 54,721 tok/s peakVRAM 13.7GB +[00:23:52] step 3900/4768 loss 3.0150 lr 4.39e-04 gnorm 0.08 54,716 tok/s peakVRAM 13.7GB +[00:27:04] step 3920/4768 loss 3.0093 lr 4.21e-04 gnorm 0.07 54,715 tok/s peakVRAM 13.7GB +[00:30:16] step 3940/4768 loss 3.0019 lr 4.05e-04 gnorm 0.07 54,715 tok/s peakVRAM 13.7GB +[00:33:27] step 3960/4768 loss 3.0024 lr 3.89e-04 gnorm 0.08 54,716 tok/s peakVRAM 13.7GB +[00:36:39] step 3980/4768 loss 2.9945 lr 3.75e-04 gnorm 0.07 54,721 tok/s peakVRAM 13.7GB +[00:39:50] step 4000/4768 loss 2.9933 lr 3.62e-04 gnorm 0.07 54,722 tok/s peakVRAM 13.7GB +[00:43:02] step 4020/4768 loss 2.9906 lr 3.50e-04 gnorm 0.07 54,713 tok/s peakVRAM 13.7GB +[00:46:14] step 4040/4768 loss 2.9812 lr 3.38e-04 gnorm 0.07 54,714 tok/s peakVRAM 13.7GB +[00:49:25] step 4060/4768 loss 2.9809 lr 3.26e-04 gnorm 0.07 54,719 tok/s peakVRAM 13.7GB +[00:52:37] step 4080/4768 loss 2.9725 lr 3.15e-04 gnorm 0.07 54,715 tok/s peakVRAM 13.7GB +[00:55:49] step 4100/4768 loss 2.9776 lr 3.05e-04 gnorm 0.07 54,716 tok/s peakVRAM 13.7GB +[00:59:00] step 4120/4768 loss 2.9664 lr 2.95e-04 gnorm 0.07 54,720 tok/s peakVRAM 13.7GB +[01:02:12] step 4140/4768 loss 2.9586 lr 2.85e-04 gnorm 0.07 54,716 tok/s peakVRAM 13.7GB +[01:05:24] step 4160/4768 loss 2.9768 lr 2.75e-04 gnorm 0.07 54,710 tok/s peakVRAM 13.7GB +[01:08:35] step 4180/4768 loss 2.9694 lr 2.66e-04 gnorm 0.07 54,712 tok/s peakVRAM 13.7GB +[01:11:47] step 4200/4768 loss 2.9631 lr 2.57e-04 gnorm 0.07 54,715 tok/s peakVRAM 13.7GB +[01:14:58] step 4220/4768 loss 2.9561 lr 2.48e-04 gnorm 0.07 54,716 tok/s peakVRAM 13.7GB +[01:18:10] step 4240/4768 loss 2.9624 lr 2.40e-04 gnorm 0.07 54,718 tok/s peakVRAM 13.7GB +[01:21:22] step 4260/4768 loss 2.9546 lr 2.31e-04 gnorm 0.07 54,720 tok/s peakVRAM 13.7GB +[01:24:33] step 4280/4768 loss 2.9540 lr 2.23e-04 gnorm 0.07 54,725 tok/s peakVRAM 13.7GB +[01:27:45] step 4300/4768 loss 2.9518 lr 2.15e-04 gnorm 0.07 54,721 tok/s peakVRAM 13.7GB +[01:30:57] step 4320/4768 loss 2.9470 lr 2.07e-04 gnorm 0.07 54,718 tok/s peakVRAM 13.7GB +[01:34:08] step 4340/4768 loss 2.9457 lr 1.99e-04 gnorm 0.07 54,717 tok/s peakVRAM 13.7GB +[01:37:20] step 4360/4768 loss 2.9396 lr 1.92e-04 gnorm 0.07 54,717 tok/s peakVRAM 13.7GB +[01:40:32] step 4380/4768 loss 2.9465 lr 1.84e-04 gnorm 0.07 54,713 tok/s peakVRAM 13.7GB +[01:43:43] step 4400/4768 loss 2.9439 lr 1.77e-04 gnorm 0.07 54,711 tok/s peakVRAM 13.7GB +[01:46:55] step 4420/4768 loss 2.9371 lr 1.70e-04 gnorm 0.07 54,714 tok/s peakVRAM 13.7GB +[01:50:06] step 4440/4768 loss 2.9435 lr 1.63e-04 gnorm 0.07 54,713 tok/s peakVRAM 13.7GB +[01:53:18] step 4460/4768 loss 2.9390 lr 1.56e-04 gnorm 0.07 54,716 tok/s peakVRAM 13.7GB +[01:56:30] step 4480/4768 loss 2.9341 lr 1.49e-04 gnorm 0.07 54,719 tok/s peakVRAM 13.7GB +[01:59:41] step 4500/4768 loss 2.9313 lr 1.42e-04 gnorm 0.07 54,720 tok/s peakVRAM 13.7GB +[02:02:53] step 4520/4768 loss 2.9332 lr 1.36e-04 gnorm 0.07 54,717 tok/s peakVRAM 13.7GB +[02:06:05] step 4540/4768 loss 2.9246 lr 1.29e-04 gnorm 0.07 54,717 tok/s peakVRAM 13.7GB +[02:09:16] step 4560/4768 loss 2.9280 lr 1.23e-04 gnorm 0.07 54,712 tok/s peakVRAM 13.7GB +[02:12:28] step 4580/4768 loss 2.9309 lr 1.16e-04 gnorm 0.07 54,710 tok/s peakVRAM 13.7GB +[02:15:40] step 4600/4768 loss 2.9299 lr 1.10e-04 gnorm 0.07 54,708 tok/s peakVRAM 13.7GB +[02:18:51] step 4620/4768 loss 2.9163 lr 1.04e-04 gnorm 0.07 54,711 tok/s peakVRAM 13.7GB +[02:22:03] step 4640/4768 loss 2.9256 lr 9.78e-05 gnorm 0.07 54,714 tok/s peakVRAM 13.7GB +[02:25:15] step 4660/4768 loss 2.9184 lr 9.18e-05 gnorm 0.07 54,713 tok/s peakVRAM 13.7GB +[02:28:26] step 4680/4768 loss 2.9159 lr 8.58e-05 gnorm 0.07 54,711 tok/s peakVRAM 13.7GB +[02:31:38] step 4700/4768 loss 2.9121 lr 7.99e-05 gnorm 0.07 54,712 tok/s peakVRAM 13.7GB +[02:34:50] step 4720/4768 loss 2.9131 lr 7.41e-05 gnorm 0.07 54,715 tok/s peakVRAM 13.7GB +[02:38:01] step 4740/4768 loss 2.9110 lr 6.83e-05 gnorm 0.07 54,718 tok/s peakVRAM 13.7GB +[02:41:13] step 4760/4768 loss 2.9202 lr 6.26e-05 gnorm 0.07 54,720 tok/s peakVRAM 13.7GB +[02:42:31] ckpt @ 4768 +[02:42:31] DONE diff --git a/R2/run/val_en.json b/R2/run/val_en.json new file mode 100644 index 0000000000000000000000000000000000000000..db15792515fd63f42bf46a52d67de89f328118d0 --- /dev/null +++ b/R2/run/val_en.json @@ -0,0 +1 @@ +{"schema": "val_bpb_v0", "ckpt_sha256": "f4a06ade988a35bfa2d47f686253907400eff9eacd62c91818457fd863d48394", "step": 4768, "val_sha256": "046bc03b426b75f0e84db7cdf3965a9653d94e722b81b0d03fd0cb92884cf0c5", "tokenizer_sha256": "0640d3bd3674d7a6e59c540d945a2ad275a12e1bcaeb6007c37a90751f9815dd", "script_sha256": "ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1", "trainer_sha256": "db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b", "window": 4096, "n_windows": 547, "tokens": 2240512, "bytes": 9799872, "sum_nat": 6797384.25, "bpb": 1.0006816975254111, "ce_token": 3.0338530880441614, "P": 183561103, "dtype": "fp32, tf32 off, bez autocast", "torch": "2.6.0+cu124", "device": "NVIDIA GeForce RTX 4090"} \ No newline at end of file diff --git a/R2/run/val_en.out b/R2/run/val_en.out new file mode 100644 index 0000000000000000000000000000000000000000..50daeb658b011173d63cb75ac937192d895a033c --- /dev/null +++ b/R2/run/val_en.out @@ -0,0 +1 @@ +VAL-BPB-OK step 4768 bpb 1.00068 ce 3.03385 bajty 9799872 P 183561103 -> /home/ubuntu/kolektyw/dowody400m/wyniki/R2/val_en.json diff --git a/R2/run/val_pl.json b/R2/run/val_pl.json new file mode 100644 index 0000000000000000000000000000000000000000..770468e5c1ec09c19b586368bd604aefd62d22da --- /dev/null +++ b/R2/run/val_pl.json @@ -0,0 +1 @@ +{"schema": "val_bpb_v0", "ckpt_sha256": "f4a06ade988a35bfa2d47f686253907400eff9eacd62c91818457fd863d48394", "step": 4768, "val_sha256": "99e66265f47d249c72f5e76838aa23e82959b99089f720bea6e9ee2cb1f49ee1", "tokenizer_sha256": "0640d3bd3674d7a6e59c540d945a2ad275a12e1bcaeb6007c37a90751f9815dd", "script_sha256": "ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1", "trainer_sha256": "db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b", "window": 4096, "n_windows": 406, "tokens": 1662976, "bytes": 7234678, "sum_nat": 5456679.5625, "bpb": 1.0881375155243824, "ce_token": 3.2812737901809768, "P": 183561103, "dtype": "fp32, tf32 off, bez autocast", "torch": "2.6.0+cu124", "device": "NVIDIA GeForce RTX 4090"} \ No newline at end of file diff --git a/R2/run/val_pl.out b/R2/run/val_pl.out new file mode 100644 index 0000000000000000000000000000000000000000..da78f3a08dcab453ca4ce00daf753b4e8ddd69e7 --- /dev/null +++ b/R2/run/val_pl.out @@ -0,0 +1 @@ +VAL-BPB-OK step 4768 bpb 1.08814 ce 3.28127 bajty 7234678 P 183561103 -> /home/ubuntu/kolektyw/dowody400m/wyniki/R2/val_pl.json diff --git a/R2/run/wynik.json b/R2/run/wynik.json new file mode 100644 index 0000000000000000000000000000000000000000..7ad54d5a61fdace4abc954c112f3e16de91c043e --- /dev/null +++ b/R2/run/wynik.json @@ -0,0 +1 @@ +{"ramie": "R2", "ckpt_sha256": "f4a06ade988a35bfa2d47f686253907400eff9eacd62c91818457fd863d48394", "bpb_pl": 1.0881375155243824, "bpb_en": 1.0006816975254111, "bpb_wazona_40_60": 1.0356640247249995, "val_pl_sha256": "99e66265f47d249c72f5e76838aa23e82959b99089f720bea6e9ee2cb1f49ee1", "val_en_sha256": "046bc03b426b75f0e84db7cdf3965a9653d94e722b81b0d03fd0cb92884cf0c5", "P": 183561103, "step": 4768} diff --git a/R3/run/args.txt b/R3/run/args.txt new file mode 100644 index 0000000000000000000000000000000000000000..63423dfc2a47708092aa4afa48ea77da6f085b2a --- /dev/null +++ b/R3/run/args.txt @@ -0,0 +1 @@ +--data-dir /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2 --out-dir /home/ubuntu/kolektyw/dowody400m/wyniki/R3 --run-id dowody_R3 --vocab 32768 --dtype uint16 --block 4096 --n-layer 16 --n-embd 896 --n-head 14 --value-residual --batch 128 --grad-accum 8 --norm rmsnorm --pos rope --rope-theta 500000 --ffn swiglu --ffn-mult 2.667 --qk-norm --attn-bias 0 --optimizer muon --muon-lr 0.02 --lr 6e-4 --min-lr 6e-05 --wd 0.1 --schedule wsd --decay-frac 0.2 --warmup 500 --act-ckpt --loss-chunk 1024 --sampler random --seed 3 --data-seed 20261022 --compile --log-every 20 --eval-every 4769 --eval-iters 1 --steps 4768 --ckpt-every 4768 \ No newline at end of file diff --git a/R3/run/metrics.jsonl b/R3/run/metrics.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..080cdf09253cb48c7ec3992531095710d0c3869d --- /dev/null +++ b/R3/run/metrics.jsonl @@ -0,0 +1,238 @@ +{"step": 20, "loss": 10.391170692443847, "lr": 2.3999999999999997e-05, "tok_s": 39938.219579502} +{"step": 40, "loss": 9.551794004440307, "lr": 4.7999999999999994e-05, "tok_s": 54379.42828131193} +{"step": 60, "loss": 8.678150033950805, "lr": 7.199999999999999e-05, "tok_s": 54389.02115452951} +{"step": 80, "loss": 8.038501811027526, "lr": 9.599999999999999e-05, "tok_s": 54383.87403055262} +{"step": 100, "loss": 7.597442603111267, "lr": 0.00011999999999999999, "tok_s": 54364.32824925729} +{"step": 120, "loss": 7.126488566398621, "lr": 0.00014399999999999998, "tok_s": 54354.43843397438} +{"step": 140, "loss": 6.708210778236389, "lr": 0.000168, "tok_s": 54350.018383952636} +{"step": 160, "loss": 6.35371835231781, "lr": 0.00019199999999999998, "tok_s": 54345.080699472215} +{"step": 180, "loss": 6.089379286766052, "lr": 0.00021599999999999996, "tok_s": 54335.492846580695} +{"step": 200, "loss": 5.867572855949402, "lr": 0.00023999999999999998, "tok_s": 54330.07564113113} +{"step": 220, "loss": 5.6844851732254025, "lr": 0.00026399999999999997, "tok_s": 54330.183361219075} +{"step": 240, "loss": 5.513300824165344, "lr": 0.00028799999999999995, "tok_s": 54330.08127880739} +{"step": 260, "loss": 5.349753189086914, "lr": 0.000312, "tok_s": 54328.776456418374} +{"step": 280, "loss": 5.191785573959351, "lr": 0.000336, "tok_s": 54330.021210769395} +{"step": 300, "loss": 5.046564984321594, "lr": 0.00035999999999999997, "tok_s": 54327.83750897972} +{"step": 320, "loss": 4.912378311157227, "lr": 0.00038399999999999996, "tok_s": 54328.621226859716} +{"step": 340, "loss": 4.766269516944885, "lr": 0.000408, "tok_s": 54325.99869950765} +{"step": 360, "loss": 4.637902355194091, "lr": 0.00043199999999999993, "tok_s": 54329.3188608295} +{"step": 380, "loss": 4.520739889144897, "lr": 0.00045599999999999997, "tok_s": 54328.34680894113} +{"step": 400, "loss": 4.414652609825135, "lr": 0.00047999999999999996, "tok_s": 54326.44246925295} +{"step": 420, "loss": 4.334438323974609, "lr": 0.000504, "tok_s": 54324.95127597486} +{"step": 440, "loss": 4.252598929405212, "lr": 0.0005279999999999999, "tok_s": 54324.31931173085} +{"step": 460, "loss": 4.1822011232376095, "lr": 0.000552, "tok_s": 54322.153781500776} +{"step": 480, "loss": 4.132990932464599, "lr": 0.0005759999999999999, "tok_s": 54319.78051197156} +{"step": 500, "loss": 4.076587724685669, "lr": 0.0006, "tok_s": 54319.779103086774} +{"step": 520, "loss": 4.038938069343567, "lr": 0.0006, "tok_s": 54316.490294325056} +{"step": 540, "loss": 3.989194095134735, "lr": 0.0006, "tok_s": 54316.47526804496} +{"step": 560, "loss": 3.9360882163047792, "lr": 0.0006, "tok_s": 54314.121007561145} +{"step": 580, "loss": 3.8940705180168154, "lr": 0.0006, "tok_s": 54311.014704901194} +{"step": 600, "loss": 3.851948928833008, "lr": 0.0006, "tok_s": 54311.61832452746} +{"step": 620, "loss": 3.829789924621582, "lr": 0.0006, "tok_s": 54313.57669310961} +{"step": 640, "loss": 3.8032146692276, "lr": 0.0006, "tok_s": 54308.218980855454} +{"step": 660, "loss": 3.771019697189331, "lr": 0.0006, "tok_s": 54308.99066501794} +{"step": 680, "loss": 3.7482988238334656, "lr": 0.0006, "tok_s": 54309.36139238266} +{"step": 700, "loss": 3.71762501001358, "lr": 0.0006, "tok_s": 54303.55829234693} +{"step": 720, "loss": 3.6873801589012145, "lr": 0.0006, "tok_s": 54296.27619784596} +{"step": 740, "loss": 3.6829261302948, "lr": 0.0006, "tok_s": 54300.70950058057} +{"step": 760, "loss": 3.6580036163330076, "lr": 0.0006, "tok_s": 54297.58220798803} +{"step": 780, "loss": 3.631509006023407, "lr": 0.0006, "tok_s": 54301.38550006425} +{"step": 800, "loss": 3.6185802578926087, "lr": 0.0006, "tok_s": 54303.21868780342} +{"step": 820, "loss": 3.602202534675598, "lr": 0.0006, "tok_s": 54305.088878703726} +{"step": 840, "loss": 3.5873721837997437, "lr": 0.0006, "tok_s": 54300.110145880135} +{"step": 860, "loss": 3.567150688171387, "lr": 0.0006, "tok_s": 54296.244961088916} +{"step": 880, "loss": 3.5611948490142824, "lr": 0.0006, "tok_s": 54301.182624703666} +{"step": 900, "loss": 3.5427696824073793, "lr": 0.0006, "tok_s": 54305.958441991075} +{"step": 920, "loss": 3.535680866241455, "lr": 0.0006, "tok_s": 54304.24757188373} +{"step": 940, "loss": 3.5316370844841005, "lr": 0.0006, "tok_s": 54307.76524858527} +{"step": 960, "loss": 3.5057076692581175, "lr": 0.0006, "tok_s": 54308.951433130955} +{"step": 980, "loss": 3.5031279802322386, "lr": 0.0006, "tok_s": 54307.0997525387} +{"step": 1000, "loss": 3.492707443237305, "lr": 0.0006, "tok_s": 54301.68197167505} +{"step": 1020, "loss": 3.4872264862060547, "lr": 0.0006, "tok_s": 54302.689610491594} +{"step": 1040, "loss": 3.460422468185425, "lr": 0.0006, "tok_s": 54305.59949565819} +{"step": 1060, "loss": 3.4720643639564512, "lr": 0.0006, "tok_s": 54302.2536708707} +{"step": 1080, "loss": 3.461215388774872, "lr": 0.0006, "tok_s": 54303.08137210809} +{"step": 1100, "loss": 3.4462118268013002, "lr": 0.0006, "tok_s": 54299.88757009874} +{"step": 1120, "loss": 3.428630292415619, "lr": 0.0006, "tok_s": 54298.90812426092} +{"step": 1140, "loss": 3.422501468658447, "lr": 0.0006, "tok_s": 54295.85946492801} +{"step": 1160, "loss": 3.4164438366889955, "lr": 0.0006, "tok_s": 54303.36009419397} +{"step": 1180, "loss": 3.419240880012512, "lr": 0.0006, "tok_s": 54302.53620614797} +{"step": 1200, "loss": 3.4006384015083313, "lr": 0.0006, "tok_s": 54300.54390573316} +{"step": 1220, "loss": 3.398448371887207, "lr": 0.0006, "tok_s": 54291.331451122074} +{"step": 1240, "loss": 3.3900925159454345, "lr": 0.0006, "tok_s": 54296.24563140522} +{"step": 1260, "loss": 3.3762528896331787, "lr": 0.0006, "tok_s": 54303.00540625275} +{"step": 1280, "loss": 3.3791723012924195, "lr": 0.0006, "tok_s": 54301.33313849921} +{"step": 1300, "loss": 3.372145450115204, "lr": 0.0006, "tok_s": 54296.638104439124} +{"step": 1320, "loss": 3.364592945575714, "lr": 0.0006, "tok_s": 54295.35921959387} +{"step": 1340, "loss": 3.3652435421943663, "lr": 0.0006, "tok_s": 54296.845034862825} +{"step": 1360, "loss": 3.35611697435379, "lr": 0.0006, "tok_s": 54297.074289068936} +{"step": 1380, "loss": 3.348216247558594, "lr": 0.0006, "tok_s": 54301.004758028925} +{"step": 1400, "loss": 3.3374519109725953, "lr": 0.0006, "tok_s": 54297.57409676262} +{"step": 1420, "loss": 3.325634753704071, "lr": 0.0006, "tok_s": 54301.189463174276} +{"step": 1440, "loss": 3.3325812935829164, "lr": 0.0006, "tok_s": 54303.01606693807} +{"step": 1460, "loss": 3.330060291290283, "lr": 0.0006, "tok_s": 54296.05780942607} +{"step": 1480, "loss": 3.33405921459198, "lr": 0.0006, "tok_s": 54298.77371299319} +{"step": 1500, "loss": 3.3076577305793764, "lr": 0.0006, "tok_s": 54299.24821120346} +{"step": 1520, "loss": 3.314529776573181, "lr": 0.0006, "tok_s": 54297.43754699285} +{"step": 1540, "loss": 3.298666608333588, "lr": 0.0006, "tok_s": 54295.43469485042} +{"step": 1560, "loss": 3.3075270891189574, "lr": 0.0006, "tok_s": 54296.42929869427} +{"step": 1580, "loss": 3.2952563166618347, "lr": 0.0006, "tok_s": 54291.6753973621} +{"step": 1600, "loss": 3.2850209951400755, "lr": 0.0006, "tok_s": 54289.51126259675} +{"step": 1620, "loss": 3.285892629623413, "lr": 0.0006, "tok_s": 54295.92850661321} +{"step": 1640, "loss": 3.282339608669281, "lr": 0.0006, "tok_s": 54293.70176615149} +{"step": 1660, "loss": 3.2832374215126037, "lr": 0.0006, "tok_s": 54295.494753463805} +{"step": 1680, "loss": 3.2723647713661195, "lr": 0.0006, "tok_s": 54296.26031133628} +{"step": 1700, "loss": 3.2651630640029907, "lr": 0.0006, "tok_s": 54297.15808129404} +{"step": 1720, "loss": 3.267287790775299, "lr": 0.0006, "tok_s": 54291.46100011983} +{"step": 1740, "loss": 3.26126115322113, "lr": 0.0006, "tok_s": 54300.88528698527} +{"step": 1760, "loss": 3.2604873180389404, "lr": 0.0006, "tok_s": 54296.986877295334} +{"step": 1780, "loss": 3.2473698019981385, "lr": 0.0006, "tok_s": 54296.687842673615} +{"step": 1800, "loss": 3.251227557659149, "lr": 0.0006, "tok_s": 54296.058948955935} +{"step": 1820, "loss": 3.24706484079361, "lr": 0.0006, "tok_s": 54296.81621064058} +{"step": 1840, "loss": 3.2405392050743105, "lr": 0.0006, "tok_s": 54296.64179123236} +{"step": 1860, "loss": 3.2397434592247008, "lr": 0.0006, "tok_s": 54290.823448084586} +{"step": 1880, "loss": 3.235514509677887, "lr": 0.0006, "tok_s": 54290.13812817972} +{"step": 1900, "loss": 3.2295166730880736, "lr": 0.0006, "tok_s": 54291.516894705026} +{"step": 1920, "loss": 3.230223202705383, "lr": 0.0006, "tok_s": 54291.39813560397} +{"step": 1940, "loss": 3.2223541855812075, "lr": 0.0006, "tok_s": 54290.18323035624} +{"step": 1960, "loss": 3.2265710711479185, "lr": 0.0006, "tok_s": 54291.84167535139} +{"step": 1980, "loss": 3.22081196308136, "lr": 0.0006, "tok_s": 54293.433129879304} +{"step": 2000, "loss": 3.21427800655365, "lr": 0.0006, "tok_s": 54302.276936004324} +{"step": 2020, "loss": 3.2142665863037108, "lr": 0.0006, "tok_s": 54302.98629748764} +{"step": 2040, "loss": 3.2073814868927, "lr": 0.0006, "tok_s": 54302.76302764977} +{"step": 2060, "loss": 3.207506799697876, "lr": 0.0006, "tok_s": 54298.39870572258} +{"step": 2080, "loss": 3.196396732330322, "lr": 0.0006, "tok_s": 54300.38937380077} +{"step": 2100, "loss": 3.204671251773834, "lr": 0.0006, "tok_s": 54298.56582934546} +{"step": 2120, "loss": 3.1943978548049925, "lr": 0.0006, "tok_s": 54296.33089577319} +{"step": 2140, "loss": 3.2071911692619324, "lr": 0.0006, "tok_s": 54297.979526929994} +{"step": 2160, "loss": 3.186479187011719, "lr": 0.0006, "tok_s": 54300.04531714343} +{"step": 2180, "loss": 3.1878172636032103, "lr": 0.0006, "tok_s": 54300.669945444235} +{"step": 2200, "loss": 3.188826882839203, "lr": 0.0006, "tok_s": 54296.53896340529} +{"step": 2220, "loss": 3.180976092815399, "lr": 0.0006, "tok_s": 54295.00208907626} +{"step": 2240, "loss": 3.178165864944458, "lr": 0.0006, "tok_s": 54292.85270230039} +{"step": 2260, "loss": 3.174638867378235, "lr": 0.0006, "tok_s": 54296.32291898528} +{"step": 2280, "loss": 3.171221601963043, "lr": 0.0006, "tok_s": 54291.15820627391} +{"step": 2300, "loss": 3.174256992340088, "lr": 0.0006, "tok_s": 54296.46288178916} +{"step": 2320, "loss": 3.170739507675171, "lr": 0.0006, "tok_s": 54296.68516136485} +{"step": 2340, "loss": 3.165570116043091, "lr": 0.0006, "tok_s": 54298.704462958885} +{"step": 2360, "loss": 3.1563900113105774, "lr": 0.0006, "tok_s": 54294.17872272945} +{"step": 2380, "loss": 3.15545539855957, "lr": 0.0006, "tok_s": 54296.46308288566} +{"step": 2400, "loss": 3.1550696969032286, "lr": 0.0006, "tok_s": 54296.34309557102} +{"step": 2420, "loss": 3.1582520246505736, "lr": 0.0006, "tok_s": 54299.31920564437} +{"step": 2440, "loss": 3.1520421862602235, "lr": 0.0006, "tok_s": 54301.88371099248} +{"step": 2460, "loss": 3.1509958863258363, "lr": 0.0006, "tok_s": 54302.544117714424} +{"step": 2480, "loss": 3.1482174158096314, "lr": 0.0006, "tok_s": 54302.01579105181} +{"step": 2500, "loss": 3.1425801277160645, "lr": 0.0006, "tok_s": 54300.24697725865} +{"step": 2520, "loss": 3.144414746761322, "lr": 0.0006, "tok_s": 54301.547077361196} +{"step": 2540, "loss": 3.1448075413703918, "lr": 0.0006, "tok_s": 54301.01072489057} +{"step": 2560, "loss": 3.141619086265564, "lr": 0.0006, "tok_s": 54302.40238033928} +{"step": 2580, "loss": 3.136859822273254, "lr": 0.0006, "tok_s": 54297.256822384035} +{"step": 2600, "loss": 3.1276493072509766, "lr": 0.0006, "tok_s": 54297.72713789518} +{"step": 2620, "loss": 3.1361867904663088, "lr": 0.0006, "tok_s": 54300.79806406936} +{"step": 2640, "loss": 3.137372374534607, "lr": 0.0006, "tok_s": 54303.13514506621} +{"step": 2660, "loss": 3.1361696124076843, "lr": 0.0006, "tok_s": 54300.98531545514} +{"step": 2680, "loss": 3.130984938144684, "lr": 0.0006, "tok_s": 54300.93255240265} +{"step": 2700, "loss": 3.1338071823120117, "lr": 0.0006, "tok_s": 54298.61201812545} +{"step": 2720, "loss": 3.1165393590927124, "lr": 0.0006, "tok_s": 54301.79360191969} +{"step": 2740, "loss": 3.1183090806007385, "lr": 0.0006, "tok_s": 54302.1752936652} +{"step": 2760, "loss": 3.121435749530792, "lr": 0.0006, "tok_s": 54301.08179098398} +{"step": 2780, "loss": 3.112166333198547, "lr": 0.0006, "tok_s": 54299.28119443337} +{"step": 2800, "loss": 3.1070509791374206, "lr": 0.0006, "tok_s": 54294.299437773414} +{"step": 2820, "loss": 3.1044181108474733, "lr": 0.0006, "tok_s": 54293.94138283397} +{"step": 2840, "loss": 3.1123920321464538, "lr": 0.0006, "tok_s": 54299.004592408884} +{"step": 2860, "loss": 3.104003381729126, "lr": 0.0006, "tok_s": 54295.7829160027} +{"step": 2880, "loss": 3.105788505077362, "lr": 0.0006, "tok_s": 54297.402353864534} +{"step": 2900, "loss": 3.1077197432518004, "lr": 0.0006, "tok_s": 54301.095870137775} +{"step": 2920, "loss": 3.1145001649856567, "lr": 0.0006, "tok_s": 54297.948623392236} +{"step": 2940, "loss": 3.1046595215797423, "lr": 0.0006, "tok_s": 54296.37842139013} +{"step": 2960, "loss": 3.1066341519355776, "lr": 0.0006, "tok_s": 54297.438954718935} +{"step": 2980, "loss": 3.09440678358078, "lr": 0.0006, "tok_s": 54299.82220556979} +{"step": 3000, "loss": 3.094467115402222, "lr": 0.0006, "tok_s": 54294.68055497727} +{"step": 3020, "loss": 3.0900529623031616, "lr": 0.0006, "tok_s": 54295.913156553455} +{"step": 3040, "loss": 3.0873758673667906, "lr": 0.0006, "tok_s": 54294.93646819516} +{"step": 3060, "loss": 3.091061735153198, "lr": 0.0006, "tok_s": 54303.65940346399} +{"step": 3080, "loss": 3.0834268808364866, "lr": 0.0006, "tok_s": 54304.24026328324} +{"step": 3100, "loss": 3.0774672627449036, "lr": 0.0006, "tok_s": 54303.19421498459} +{"step": 3120, "loss": 3.0931316614151, "lr": 0.0006, "tok_s": 54302.30509555873} +{"step": 3140, "loss": 3.0784556031227113, "lr": 0.0006, "tok_s": 54302.14384898168} +{"step": 3160, "loss": 3.0747018098831176, "lr": 0.0006, "tok_s": 54303.67737285311} +{"step": 3180, "loss": 3.08034588098526, "lr": 0.0006, "tok_s": 54302.56309207388} +{"step": 3200, "loss": 3.082012939453125, "lr": 0.0006, "tok_s": 54305.79013291984} +{"step": 3220, "loss": 3.091588723659515, "lr": 0.0006, "tok_s": 54302.40962139822} +{"step": 3240, "loss": 3.073459792137146, "lr": 0.0006, "tok_s": 54301.934397477344} +{"step": 3260, "loss": 3.0700947999954225, "lr": 0.0006, "tok_s": 54302.64904675875} +{"step": 3280, "loss": 3.065794324874878, "lr": 0.0006, "tok_s": 54301.06616983618} +{"step": 3300, "loss": 3.076319682598114, "lr": 0.0006, "tok_s": 54302.34458603065} +{"step": 3320, "loss": 3.0704444527626036, "lr": 0.0006, "tok_s": 54303.2898267253} +{"step": 3340, "loss": 3.0707499027252196, "lr": 0.0006, "tok_s": 54304.973345825405} +{"step": 3360, "loss": 3.0617035150527956, "lr": 0.0006, "tok_s": 54304.203988515415} +{"step": 3380, "loss": 3.0714889407157897, "lr": 0.0006, "tok_s": 54304.25521574323} +{"step": 3400, "loss": 3.056394910812378, "lr": 0.0006, "tok_s": 54306.48476661177} +{"step": 3420, "loss": 3.064664351940155, "lr": 0.0006, "tok_s": 54303.1279708624} +{"step": 3440, "loss": 3.0560283184051515, "lr": 0.0006, "tok_s": 54305.83928442523} +{"step": 3460, "loss": 3.065650928020477, "lr": 0.0006, "tok_s": 54306.93022934865} +{"step": 3480, "loss": 3.065195178985596, "lr": 0.0006, "tok_s": 54303.66282301101} +{"step": 3500, "loss": 3.051303768157959, "lr": 0.0006, "tok_s": 54307.53999472173} +{"step": 3520, "loss": 3.0544704675674437, "lr": 0.0006, "tok_s": 54303.99163793554} +{"step": 3540, "loss": 3.056144821643829, "lr": 0.0006, "tok_s": 54304.83742936573} +{"step": 3560, "loss": 3.060632956027985, "lr": 0.0006, "tok_s": 54302.718843235816} +{"step": 3580, "loss": 3.0543795585632325, "lr": 0.0006, "tok_s": 54306.81421923123} +{"step": 3600, "loss": 3.0567708373069764, "lr": 0.0006, "tok_s": 54308.53611510947} +{"step": 3620, "loss": 3.0478878378868104, "lr": 0.0006, "tok_s": 54307.18786746294} +{"step": 3640, "loss": 3.0562233328819275, "lr": 0.0006, "tok_s": 54307.504050842545} +{"step": 3660, "loss": 3.045197367668152, "lr": 0.0006, "tok_s": 54303.70137675713} +{"step": 3680, "loss": 3.0520368814468384, "lr": 0.0006, "tok_s": 54305.09893672568} +{"step": 3700, "loss": 3.049117958545685, "lr": 0.0006, "tok_s": 54305.996663718346} +{"step": 3720, "loss": 3.0422117710113525, "lr": 0.0006, "tok_s": 54305.14433197779} +{"step": 3740, "loss": 3.038640928268433, "lr": 0.0006, "tok_s": 54297.09842120325} +{"step": 3760, "loss": 3.0374231934547424, "lr": 0.0006, "tok_s": 54300.85055868848} +{"step": 3780, "loss": 3.0376185536384583, "lr": 0.0006, "tok_s": 54299.502759648596} +{"step": 3800, "loss": 3.0387979626655577, "lr": 0.0006, "tok_s": 54300.89192425168} +{"step": 3820, "loss": 3.0327330589294434, "lr": 0.0005609064981513237, "tok_s": 54301.797758730456} +{"step": 3840, "loss": 3.030589687824249, "lr": 0.0005125842723878462, "tok_s": 54307.59753185125} +{"step": 3860, "loss": 3.023517918586731, "lr": 0.0004827194944539712, "tok_s": 54307.045904670194} +{"step": 3880, "loss": 3.0207011818885805, "lr": 0.0004590463745471515, "tok_s": 54306.57837822574} +{"step": 3900, "loss": 3.0139504432678224, "lr": 0.0004388133626026285, "tok_s": 54307.97534861857} +{"step": 3920, "loss": 3.008868193626404, "lr": 0.000420851068597545, "tok_s": 54310.168721040885} +{"step": 3940, "loss": 3.0007791757583617, "lr": 0.00040453249075661876, "tok_s": 54311.59250274652} +{"step": 3960, "loss": 3.0011908411979675, "lr": 0.00038947504965686095, "tok_s": 54308.519014309815} +{"step": 3980, "loss": 2.993467652797699, "lr": 0.000375424929557729, "tok_s": 54309.820583601206} +{"step": 4000, "loss": 2.9921262383461, "lr": 0.00036220351177672794, "tok_s": 54306.32369638092} +{"step": 4020, "loss": 2.9891883373260497, "lr": 0.0003496794507866152, "tok_s": 54308.82300777772} +{"step": 4040, "loss": 2.9804091453552246, "lr": 0.0003377528171635386, "tok_s": 54307.29019915697} +{"step": 4060, "loss": 2.9800134897232056, "lr": 0.00032634548705926626, "tok_s": 54306.674203055976} +{"step": 4080, "loss": 2.9711027264595034, "lr": 0.00031539501058484577, "tok_s": 54306.03032570517} +{"step": 4100, "loss": 2.977076065540314, "lr": 0.00030485053354754334, "tok_s": 54300.37120546216} +{"step": 4120, "loss": 2.9652089834213258, "lr": 0.00029466998985632013, "tok_s": 54296.37319289788} +{"step": 4140, "loss": 2.957633745670319, "lr": 0.0002848181118123862, "tok_s": 54296.02174668227} +{"step": 4160, "loss": 2.975422739982605, "lr": 0.000275264984658754, "tok_s": 54295.977774330866} +{"step": 4180, "loss": 2.9681981801986694, "lr": 0.0002659849737873323, "tok_s": 54298.950961707524} +{"step": 4200, "loss": 2.961822283267975, "lr": 0.0002569559134994179, "tok_s": 54299.268792197916} +{"step": 4220, "loss": 2.9553013324737547, "lr": 0.00024815848336191376, "tok_s": 54301.51295161963} +{"step": 4240, "loss": 2.961420774459839, "lr": 0.00023957572171486765, "tok_s": 54297.290741673656} +{"step": 4260, "loss": 2.9539794445037844, "lr": 0.0002311926411735683, "tok_s": 54296.9555056538} +{"step": 4280, "loss": 2.952929139137268, "lr": 0.0002229959211471338, "tok_s": 54295.72238753944} +{"step": 4300, "loss": 2.9505680322647097, "lr": 0.00021497365931813446, "tok_s": 54296.3422241567} +{"step": 4320, "loss": 2.945917081832886, "lr": 0.00020711516882674415, "tok_s": 54296.551565490925} +{"step": 4340, "loss": 2.9443411588668824, "lr": 0.00019941081128766396, "tok_s": 54299.93469969776} +{"step": 4360, "loss": 2.939062213897705, "lr": 0.00019185185819315875, "tok_s": 54298.39153277022} +{"step": 4380, "loss": 2.9452916502952577, "lr": 0.00018443037501815989, "tok_s": 54296.4495423912} +{"step": 4400, "loss": 2.9430905103683473, "lr": 0.00017713912364145464, "tok_s": 54300.390446470585} +{"step": 4420, "loss": 2.9360124349594114, "lr": 0.00016997147966456131, "tok_s": 54299.78265172614} +{"step": 4440, "loss": 2.9421834468841555, "lr": 0.00016292136193923112, "tok_s": 54300.294979022736} +{"step": 4460, "loss": 2.9380831718444824, "lr": 0.00015598317216998812, "tok_s": 54298.70687631626} +{"step": 4480, "loss": 2.933145081996918, "lr": 0.00014915174288526275, "tok_s": 54297.69033554667} +{"step": 4500, "loss": 2.930334281921387, "lr": 0.00014242229240208165, "tok_s": 54299.18666943039} +{"step": 4520, "loss": 2.9317137598991394, "lr": 0.00013579038566856907, "tok_s": 54299.3572169086} +{"step": 4540, "loss": 2.923252594470978, "lr": 0.00012925190007297358, "tok_s": 54293.08420160447} +{"step": 4560, "loss": 2.9264015793800353, "lr": 0.0001228029954703451, "tok_s": 54295.19493092878} +{"step": 4580, "loss": 2.929991102218628, "lr": 0.00011644008780788571, "tok_s": 54293.496267207294} +{"step": 4600, "loss": 2.9283700942993165, "lr": 0.00011015982583457378, "tok_s": 54295.239907449395} +{"step": 4620, "loss": 2.915508234500885, "lr": 0.00010395907046536478, "tok_s": 54290.12981812863} +{"step": 4640, "loss": 2.9243770480155944, "lr": 9.783487643927846e-05, "tok_s": 54291.37528190881} +{"step": 4660, "loss": 2.917322552204132, "lr": 9.178447596720887e-05, "tok_s": 54297.78163763953} +{"step": 4680, "loss": 2.915124475955963, "lr": 8.580526411184025e-05, "tok_s": 54298.7181386535} +{"step": 4700, "loss": 2.9109233140945436, "lr": 7.98947856805662e-05, "tok_s": 54299.28662460312} +{"step": 4720, "loss": 2.912209415435791, "lr": 7.405072344432876e-05, "tok_s": 54298.01746928259} +{"step": 4740, "loss": 2.9099733233451843, "lr": 6.827088752203553e-05, "tok_s": 54300.51561392098} +{"step": 4760, "loss": 2.919350802898407, "lr": 6.255320579263513e-05, "tok_s": 54312.82231747717} diff --git a/R3/run/srodowisko.txt b/R3/run/srodowisko.txt new file mode 100644 index 0000000000000000000000000000000000000000..83ca76fa28ac40d72f838fa166949bd058f09405 --- /dev/null +++ b/R3/run/srodowisko.txt @@ -0,0 +1,63 @@ +data 2026-10-22T11:59:50Z ARM R3 GPU 2 MINI 0 DATA_SEED 20261022 +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +a607076bd6e11cad879b8213b37492a4b4f5cbf8fa96205b7ebaeace8a25213e /home/ubuntu/kolektyw/dowody400m/kod/ramie.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 diff --git a/R3/run/srodowisko_R3_mini0.txt b/R3/run/srodowisko_R3_mini0.txt new file mode 100644 index 0000000000000000000000000000000000000000..83ca76fa28ac40d72f838fa166949bd058f09405 --- /dev/null +++ b/R3/run/srodowisko_R3_mini0.txt @@ -0,0 +1,63 @@ +data 2026-10-22T11:59:50Z ARM R3 GPU 2 MINI 0 DATA_SEED 20261022 +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +a607076bd6e11cad879b8213b37492a4b4f5cbf8fa96205b7ebaeace8a25213e /home/ubuntu/kolektyw/dowody400m/kod/ramie.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 diff --git a/R3/run/train.log b/R3/run/train.log new file mode 100644 index 0000000000000000000000000000000000000000..0af7b4d9109bc1f909e505147706aa7c88143d64 --- /dev/null +++ b/R3/run/train.log @@ -0,0 +1,248 @@ +[13:59:56] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[13:59:56] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[13:59:58] model GPT-ref: 183.6M param (L16 d896 h14) +[13:59:59] optimizer=muon muon_lr=0.02 (mult=33.3x) +[13:59:59] r11 pamiec: act_ckpt=True loss_chunk=1024 +[13:59:59] r13: mixer_layout=AAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[14:00:00] torch.compile enabled +[14:00:00] START gpt-ref: steps=4768 (od 0) lr=0.0006->6e-05 +[14:04:22] step 20/4768 loss 10.3912 lr 2.40e-05 gnorm 2.73 39,938 tok/s peakVRAM 13.7GB +[14:07:35] step 40/4768 loss 9.5518 lr 4.80e-05 gnorm 1.36 54,379 tok/s peakVRAM 13.7GB +[14:10:48] step 60/4768 loss 8.6782 lr 7.20e-05 gnorm 0.73 54,389 tok/s peakVRAM 13.7GB +[14:14:00] step 80/4768 loss 8.0385 lr 9.60e-05 gnorm 1.30 54,384 tok/s peakVRAM 13.7GB +[14:17:13] step 100/4768 loss 7.5974 lr 1.20e-04 gnorm 0.84 54,364 tok/s peakVRAM 13.7GB +[14:20:26] step 120/4768 loss 7.1265 lr 1.44e-04 gnorm 0.78 54,354 tok/s peakVRAM 13.7GB +[14:23:39] step 140/4768 loss 6.7082 lr 1.68e-04 gnorm 0.88 54,350 tok/s peakVRAM 13.7GB +[14:26:52] step 160/4768 loss 6.3537 lr 1.92e-04 gnorm 0.78 54,345 tok/s peakVRAM 13.7GB +[14:30:05] step 180/4768 loss 6.0894 lr 2.16e-04 gnorm 0.86 54,335 tok/s peakVRAM 13.7GB +[14:33:18] step 200/4768 loss 5.8676 lr 2.40e-04 gnorm 0.76 54,330 tok/s peakVRAM 13.7GB +[14:36:31] step 220/4768 loss 5.6845 lr 2.64e-04 gnorm 0.84 54,330 tok/s peakVRAM 13.7GB +[14:39:44] step 240/4768 loss 5.5133 lr 2.88e-04 gnorm 0.76 54,330 tok/s peakVRAM 13.7GB +[14:42:57] step 260/4768 loss 5.3498 lr 3.12e-04 gnorm 0.70 54,329 tok/s peakVRAM 13.7GB +[14:46:10] step 280/4768 loss 5.1918 lr 3.36e-04 gnorm 0.66 54,330 tok/s peakVRAM 13.7GB +[14:49:23] step 300/4768 loss 5.0466 lr 3.60e-04 gnorm 0.66 54,328 tok/s peakVRAM 13.7GB +[14:52:36] step 320/4768 loss 4.9124 lr 3.84e-04 gnorm 0.72 54,329 tok/s peakVRAM 13.7GB +[14:55:49] step 340/4768 loss 4.7663 lr 4.08e-04 gnorm 0.64 54,326 tok/s peakVRAM 13.7GB +[14:59:02] step 360/4768 loss 4.6379 lr 4.32e-04 gnorm 0.60 54,329 tok/s peakVRAM 13.7GB +[15:02:15] step 380/4768 loss 4.5207 lr 4.56e-04 gnorm 0.50 54,328 tok/s peakVRAM 13.7GB +[15:05:28] step 400/4768 loss 4.4147 lr 4.80e-04 gnorm 0.44 54,326 tok/s peakVRAM 13.7GB +[15:08:41] step 420/4768 loss 4.3344 lr 5.04e-04 gnorm 0.48 54,325 tok/s peakVRAM 13.7GB +[15:11:54] step 440/4768 loss 4.2526 lr 5.28e-04 gnorm 0.36 54,324 tok/s peakVRAM 13.7GB +[15:15:07] step 460/4768 loss 4.1822 lr 5.52e-04 gnorm 0.47 54,322 tok/s peakVRAM 13.7GB +[15:18:20] step 480/4768 loss 4.1330 lr 5.76e-04 gnorm 0.38 54,320 tok/s peakVRAM 13.7GB +[15:21:33] step 500/4768 loss 4.0766 lr 6.00e-04 gnorm 0.30 54,320 tok/s peakVRAM 13.7GB +[15:24:46] step 520/4768 loss 4.0389 lr 6.00e-04 gnorm 0.35 54,316 tok/s peakVRAM 13.7GB +[15:27:59] step 540/4768 loss 3.9892 lr 6.00e-04 gnorm 0.26 54,316 tok/s peakVRAM 13.7GB +[15:31:13] step 560/4768 loss 3.9361 lr 6.00e-04 gnorm 0.24 54,314 tok/s peakVRAM 13.7GB +[15:34:26] step 580/4768 loss 3.8941 lr 6.00e-04 gnorm 0.24 54,311 tok/s peakVRAM 13.7GB +[15:37:39] step 600/4768 loss 3.8519 lr 6.00e-04 gnorm 0.22 54,312 tok/s peakVRAM 13.7GB +[15:40:52] step 620/4768 loss 3.8298 lr 6.00e-04 gnorm 0.22 54,314 tok/s peakVRAM 13.7GB +[15:44:05] step 640/4768 loss 3.8032 lr 6.00e-04 gnorm 0.19 54,308 tok/s peakVRAM 13.7GB +[15:47:18] step 660/4768 loss 3.7710 lr 6.00e-04 gnorm 0.18 54,309 tok/s peakVRAM 13.7GB +[15:50:31] step 680/4768 loss 3.7483 lr 6.00e-04 gnorm 0.18 54,309 tok/s peakVRAM 13.7GB +[15:53:44] step 700/4768 loss 3.7176 lr 6.00e-04 gnorm 0.17 54,304 tok/s peakVRAM 13.7GB +[15:56:57] step 720/4768 loss 3.6874 lr 6.00e-04 gnorm 0.18 54,296 tok/s peakVRAM 13.7GB +[16:00:10] step 740/4768 loss 3.6829 lr 6.00e-04 gnorm 0.18 54,301 tok/s peakVRAM 13.7GB +[16:03:23] step 760/4768 loss 3.6580 lr 6.00e-04 gnorm 0.15 54,298 tok/s peakVRAM 13.7GB +[16:06:36] step 780/4768 loss 3.6315 lr 6.00e-04 gnorm 0.16 54,301 tok/s peakVRAM 13.7GB +[16:09:50] step 800/4768 loss 3.6186 lr 6.00e-04 gnorm 0.17 54,303 tok/s peakVRAM 13.7GB +[16:13:03] step 820/4768 loss 3.6022 lr 6.00e-04 gnorm 0.15 54,305 tok/s peakVRAM 13.7GB +[16:16:16] step 840/4768 loss 3.5874 lr 6.00e-04 gnorm 0.17 54,300 tok/s peakVRAM 13.7GB +[16:19:29] step 860/4768 loss 3.5672 lr 6.00e-04 gnorm 0.16 54,296 tok/s peakVRAM 13.7GB +[16:22:42] step 880/4768 loss 3.5612 lr 6.00e-04 gnorm 0.13 54,301 tok/s peakVRAM 13.7GB +[16:25:55] step 900/4768 loss 3.5428 lr 6.00e-04 gnorm 0.14 54,306 tok/s peakVRAM 13.7GB +[16:29:08] step 920/4768 loss 3.5357 lr 6.00e-04 gnorm 0.14 54,304 tok/s peakVRAM 13.7GB +[16:32:21] step 940/4768 loss 3.5316 lr 6.00e-04 gnorm 0.13 54,308 tok/s peakVRAM 13.7GB +[16:35:34] step 960/4768 loss 3.5057 lr 6.00e-04 gnorm 0.13 54,309 tok/s peakVRAM 13.7GB +[16:38:47] step 980/4768 loss 3.5031 lr 6.00e-04 gnorm 0.14 54,307 tok/s peakVRAM 13.7GB +[16:42:01] step 1000/4768 loss 3.4927 lr 6.00e-04 gnorm 0.13 54,302 tok/s peakVRAM 13.7GB +[16:45:14] step 1020/4768 loss 3.4872 lr 6.00e-04 gnorm 0.13 54,303 tok/s peakVRAM 13.7GB +[16:48:27] step 1040/4768 loss 3.4604 lr 6.00e-04 gnorm 0.13 54,306 tok/s peakVRAM 13.7GB +[16:51:40] step 1060/4768 loss 3.4721 lr 6.00e-04 gnorm 0.13 54,302 tok/s peakVRAM 13.7GB +[16:54:53] step 1080/4768 loss 3.4612 lr 6.00e-04 gnorm 0.13 54,303 tok/s peakVRAM 13.7GB +[16:58:06] step 1100/4768 loss 3.4462 lr 6.00e-04 gnorm 0.14 54,300 tok/s peakVRAM 13.7GB +[17:01:19] step 1120/4768 loss 3.4286 lr 6.00e-04 gnorm 0.13 54,299 tok/s peakVRAM 13.7GB +[17:04:32] step 1140/4768 loss 3.4225 lr 6.00e-04 gnorm 0.12 54,296 tok/s peakVRAM 13.7GB +[17:07:45] step 1160/4768 loss 3.4164 lr 6.00e-04 gnorm 0.12 54,303 tok/s peakVRAM 13.7GB +[17:10:58] step 1180/4768 loss 3.4192 lr 6.00e-04 gnorm 0.12 54,303 tok/s peakVRAM 13.7GB +[17:14:12] step 1200/4768 loss 3.4006 lr 6.00e-04 gnorm 0.12 54,301 tok/s peakVRAM 13.7GB +[17:17:25] step 1220/4768 loss 3.3984 lr 6.00e-04 gnorm 0.11 54,291 tok/s peakVRAM 13.7GB +[17:20:38] step 1240/4768 loss 3.3901 lr 6.00e-04 gnorm 0.12 54,296 tok/s peakVRAM 13.7GB +[17:23:51] step 1260/4768 loss 3.3763 lr 6.00e-04 gnorm 0.13 54,303 tok/s peakVRAM 13.7GB +[17:27:04] step 1280/4768 loss 3.3792 lr 6.00e-04 gnorm 0.12 54,301 tok/s peakVRAM 13.7GB +[17:30:17] step 1300/4768 loss 3.3721 lr 6.00e-04 gnorm 0.13 54,297 tok/s peakVRAM 13.7GB +[17:33:30] step 1320/4768 loss 3.3646 lr 6.00e-04 gnorm 0.11 54,295 tok/s peakVRAM 13.7GB +[17:36:43] step 1340/4768 loss 3.3652 lr 6.00e-04 gnorm 0.13 54,297 tok/s peakVRAM 13.7GB +[17:39:57] step 1360/4768 loss 3.3561 lr 6.00e-04 gnorm 0.12 54,297 tok/s peakVRAM 13.7GB +[17:43:10] step 1380/4768 loss 3.3482 lr 6.00e-04 gnorm 0.13 54,301 tok/s peakVRAM 13.7GB +[17:46:23] step 1400/4768 loss 3.3375 lr 6.00e-04 gnorm 0.12 54,298 tok/s peakVRAM 13.7GB +[17:49:36] step 1420/4768 loss 3.3256 lr 6.00e-04 gnorm 0.11 54,301 tok/s peakVRAM 13.7GB +[17:52:49] step 1440/4768 loss 3.3326 lr 6.00e-04 gnorm 0.12 54,303 tok/s peakVRAM 13.7GB +[17:56:02] step 1460/4768 loss 3.3301 lr 6.00e-04 gnorm 0.12 54,296 tok/s peakVRAM 13.7GB +[17:59:15] step 1480/4768 loss 3.3341 lr 6.00e-04 gnorm 0.11 54,299 tok/s peakVRAM 13.7GB +[18:02:28] step 1500/4768 loss 3.3077 lr 6.00e-04 gnorm 0.12 54,299 tok/s peakVRAM 13.7GB +[18:05:41] step 1520/4768 loss 3.3145 lr 6.00e-04 gnorm 0.11 54,297 tok/s peakVRAM 13.7GB +[18:08:55] step 1540/4768 loss 3.2987 lr 6.00e-04 gnorm 0.11 54,295 tok/s peakVRAM 13.7GB +[18:12:08] step 1560/4768 loss 3.3075 lr 6.00e-04 gnorm 0.12 54,296 tok/s peakVRAM 13.7GB +[18:15:21] step 1580/4768 loss 3.2953 lr 6.00e-04 gnorm 0.12 54,292 tok/s peakVRAM 13.7GB +[18:18:34] step 1600/4768 loss 3.2850 lr 6.00e-04 gnorm 0.11 54,290 tok/s peakVRAM 13.7GB +[18:21:47] step 1620/4768 loss 3.2859 lr 6.00e-04 gnorm 0.11 54,296 tok/s peakVRAM 13.7GB +[18:25:00] step 1640/4768 loss 3.2823 lr 6.00e-04 gnorm 0.11 54,294 tok/s peakVRAM 13.7GB +[18:28:13] step 1660/4768 loss 3.2832 lr 6.00e-04 gnorm 0.11 54,295 tok/s peakVRAM 13.7GB +[18:31:26] step 1680/4768 loss 3.2724 lr 6.00e-04 gnorm 0.11 54,296 tok/s peakVRAM 13.7GB +[18:34:40] step 1700/4768 loss 3.2652 lr 6.00e-04 gnorm 0.12 54,297 tok/s peakVRAM 13.7GB +[18:37:53] step 1720/4768 loss 3.2673 lr 6.00e-04 gnorm 0.11 54,291 tok/s peakVRAM 13.7GB +[18:41:06] step 1740/4768 loss 3.2613 lr 6.00e-04 gnorm 0.10 54,301 tok/s peakVRAM 13.7GB +[18:44:19] step 1760/4768 loss 3.2605 lr 6.00e-04 gnorm 0.11 54,297 tok/s peakVRAM 13.7GB +[18:47:32] step 1780/4768 loss 3.2474 lr 6.00e-04 gnorm 0.10 54,297 tok/s peakVRAM 13.7GB +[18:50:45] step 1800/4768 loss 3.2512 lr 6.00e-04 gnorm 0.11 54,296 tok/s peakVRAM 13.7GB +[18:53:58] step 1820/4768 loss 3.2471 lr 6.00e-04 gnorm 0.11 54,297 tok/s peakVRAM 13.7GB +[18:57:11] step 1840/4768 loss 3.2405 lr 6.00e-04 gnorm 0.11 54,297 tok/s peakVRAM 13.7GB +[19:00:25] step 1860/4768 loss 3.2397 lr 6.00e-04 gnorm 0.11 54,291 tok/s peakVRAM 13.7GB +[19:03:38] step 1880/4768 loss 3.2355 lr 6.00e-04 gnorm 0.10 54,290 tok/s peakVRAM 13.7GB +[19:06:51] step 1900/4768 loss 3.2295 lr 6.00e-04 gnorm 0.11 54,292 tok/s peakVRAM 13.7GB +[19:10:04] step 1920/4768 loss 3.2302 lr 6.00e-04 gnorm 0.11 54,291 tok/s peakVRAM 13.7GB +[19:13:17] step 1940/4768 loss 3.2224 lr 6.00e-04 gnorm 0.10 54,290 tok/s peakVRAM 13.7GB +[19:16:30] step 1960/4768 loss 3.2266 lr 6.00e-04 gnorm 0.11 54,292 tok/s peakVRAM 13.7GB +[19:19:43] step 1980/4768 loss 3.2208 lr 6.00e-04 gnorm 0.09 54,293 tok/s peakVRAM 13.7GB +[19:22:56] step 2000/4768 loss 3.2143 lr 6.00e-04 gnorm 0.12 54,302 tok/s peakVRAM 13.7GB +[19:26:10] step 2020/4768 loss 3.2143 lr 6.00e-04 gnorm 0.11 54,303 tok/s peakVRAM 13.7GB +[19:29:23] step 2040/4768 loss 3.2074 lr 6.00e-04 gnorm 0.12 54,303 tok/s peakVRAM 13.7GB +[19:32:36] step 2060/4768 loss 3.2075 lr 6.00e-04 gnorm 0.10 54,298 tok/s peakVRAM 13.7GB +[19:35:49] step 2080/4768 loss 3.1964 lr 6.00e-04 gnorm 0.09 54,300 tok/s peakVRAM 13.7GB +[19:39:02] step 2100/4768 loss 3.2047 lr 6.00e-04 gnorm 0.10 54,299 tok/s peakVRAM 13.7GB +[19:42:15] step 2120/4768 loss 3.1944 lr 6.00e-04 gnorm 0.11 54,296 tok/s peakVRAM 13.7GB +[19:45:28] step 2140/4768 loss 3.2072 lr 6.00e-04 gnorm 0.11 54,298 tok/s peakVRAM 13.7GB +[19:48:41] step 2160/4768 loss 3.1865 lr 6.00e-04 gnorm 0.10 54,300 tok/s peakVRAM 13.7GB +[19:51:54] step 2180/4768 loss 3.1878 lr 6.00e-04 gnorm 0.10 54,301 tok/s peakVRAM 13.7GB +[19:55:08] step 2200/4768 loss 3.1888 lr 6.00e-04 gnorm 0.11 54,297 tok/s peakVRAM 13.7GB +[19:58:21] step 2220/4768 loss 3.1810 lr 6.00e-04 gnorm 0.10 54,295 tok/s peakVRAM 13.7GB +[20:01:34] step 2240/4768 loss 3.1782 lr 6.00e-04 gnorm 0.11 54,293 tok/s peakVRAM 13.7GB +[20:04:47] step 2260/4768 loss 3.1746 lr 6.00e-04 gnorm 0.11 54,296 tok/s peakVRAM 13.7GB +[20:08:00] step 2280/4768 loss 3.1712 lr 6.00e-04 gnorm 0.11 54,291 tok/s peakVRAM 13.7GB +[20:11:13] step 2300/4768 loss 3.1743 lr 6.00e-04 gnorm 0.11 54,296 tok/s peakVRAM 13.7GB +[20:14:26] step 2320/4768 loss 3.1707 lr 6.00e-04 gnorm 0.10 54,297 tok/s peakVRAM 13.7GB +[20:17:39] step 2340/4768 loss 3.1656 lr 6.00e-04 gnorm 0.10 54,299 tok/s peakVRAM 13.7GB +[20:20:53] step 2360/4768 loss 3.1564 lr 6.00e-04 gnorm 0.10 54,294 tok/s peakVRAM 13.7GB +[20:24:06] step 2380/4768 loss 3.1555 lr 6.00e-04 gnorm 0.10 54,296 tok/s peakVRAM 13.7GB +[20:27:19] step 2400/4768 loss 3.1551 lr 6.00e-04 gnorm 0.10 54,296 tok/s peakVRAM 13.7GB +[20:30:32] step 2420/4768 loss 3.1583 lr 6.00e-04 gnorm 0.10 54,299 tok/s peakVRAM 13.7GB +[20:33:45] step 2440/4768 loss 3.1520 lr 6.00e-04 gnorm 0.10 54,302 tok/s peakVRAM 13.7GB +[20:36:58] step 2460/4768 loss 3.1510 lr 6.00e-04 gnorm 0.11 54,303 tok/s peakVRAM 13.7GB +[20:40:11] step 2480/4768 loss 3.1482 lr 6.00e-04 gnorm 0.10 54,302 tok/s peakVRAM 13.7GB +[20:43:24] step 2500/4768 loss 3.1426 lr 6.00e-04 gnorm 0.10 54,300 tok/s peakVRAM 13.7GB +[20:46:37] step 2520/4768 loss 3.1444 lr 6.00e-04 gnorm 0.10 54,302 tok/s peakVRAM 13.7GB +[20:49:51] step 2540/4768 loss 3.1448 lr 6.00e-04 gnorm 0.11 54,301 tok/s peakVRAM 13.7GB +[20:53:04] step 2560/4768 loss 3.1416 lr 6.00e-04 gnorm 0.09 54,302 tok/s peakVRAM 13.7GB +[20:56:17] step 2580/4768 loss 3.1369 lr 6.00e-04 gnorm 0.09 54,297 tok/s peakVRAM 13.7GB +[20:59:30] step 2600/4768 loss 3.1276 lr 6.00e-04 gnorm 0.10 54,298 tok/s peakVRAM 13.7GB +[21:02:43] step 2620/4768 loss 3.1362 lr 6.00e-04 gnorm 0.10 54,301 tok/s peakVRAM 13.7GB +[21:05:56] step 2640/4768 loss 3.1374 lr 6.00e-04 gnorm 0.10 54,303 tok/s peakVRAM 13.7GB +[21:09:09] step 2660/4768 loss 3.1362 lr 6.00e-04 gnorm 0.11 54,301 tok/s peakVRAM 13.7GB +[21:12:22] step 2680/4768 loss 3.1310 lr 6.00e-04 gnorm 0.11 54,301 tok/s peakVRAM 13.7GB +[21:15:35] step 2700/4768 loss 3.1338 lr 6.00e-04 gnorm 0.09 54,299 tok/s peakVRAM 13.7GB +[21:18:48] step 2720/4768 loss 3.1165 lr 6.00e-04 gnorm 0.10 54,302 tok/s peakVRAM 13.7GB +[21:22:02] step 2740/4768 loss 3.1183 lr 6.00e-04 gnorm 0.10 54,302 tok/s peakVRAM 13.7GB +[21:25:15] step 2760/4768 loss 3.1214 lr 6.00e-04 gnorm 0.10 54,301 tok/s peakVRAM 13.7GB +[21:28:28] step 2780/4768 loss 3.1122 lr 6.00e-04 gnorm 0.10 54,299 tok/s peakVRAM 13.7GB +[21:31:41] step 2800/4768 loss 3.1071 lr 6.00e-04 gnorm 0.11 54,294 tok/s peakVRAM 13.7GB +[21:34:54] step 2820/4768 loss 3.1044 lr 6.00e-04 gnorm 0.09 54,294 tok/s peakVRAM 13.7GB +[21:38:07] step 2840/4768 loss 3.1124 lr 6.00e-04 gnorm 0.10 54,299 tok/s peakVRAM 13.7GB +[21:41:20] step 2860/4768 loss 3.1040 lr 6.00e-04 gnorm 0.09 54,296 tok/s peakVRAM 13.7GB +[21:44:33] step 2880/4768 loss 3.1058 lr 6.00e-04 gnorm 0.10 54,297 tok/s peakVRAM 13.7GB +[21:47:47] step 2900/4768 loss 3.1077 lr 6.00e-04 gnorm 0.10 54,301 tok/s peakVRAM 13.7GB +[21:51:00] step 2920/4768 loss 3.1145 lr 6.00e-04 gnorm 0.09 54,298 tok/s peakVRAM 13.7GB +[21:54:13] step 2940/4768 loss 3.1047 lr 6.00e-04 gnorm 0.08 54,296 tok/s peakVRAM 13.7GB +[21:57:26] step 2960/4768 loss 3.1066 lr 6.00e-04 gnorm 0.10 54,297 tok/s peakVRAM 13.7GB +[22:00:39] step 2980/4768 loss 3.0944 lr 6.00e-04 gnorm 0.11 54,300 tok/s peakVRAM 13.7GB +[22:03:52] step 3000/4768 loss 3.0945 lr 6.00e-04 gnorm 0.09 54,295 tok/s peakVRAM 13.7GB +[22:07:05] step 3020/4768 loss 3.0901 lr 6.00e-04 gnorm 0.09 54,296 tok/s peakVRAM 13.7GB +[22:10:18] step 3040/4768 loss 3.0874 lr 6.00e-04 gnorm 0.10 54,295 tok/s peakVRAM 13.7GB +[22:13:31] step 3060/4768 loss 3.0911 lr 6.00e-04 gnorm 0.10 54,304 tok/s peakVRAM 13.7GB +[22:16:45] step 3080/4768 loss 3.0834 lr 6.00e-04 gnorm 0.09 54,304 tok/s peakVRAM 13.7GB +[22:19:58] step 3100/4768 loss 3.0775 lr 6.00e-04 gnorm 0.10 54,303 tok/s peakVRAM 13.7GB +[22:23:11] step 3120/4768 loss 3.0931 lr 6.00e-04 gnorm 0.10 54,302 tok/s peakVRAM 13.7GB +[22:26:24] step 3140/4768 loss 3.0785 lr 6.00e-04 gnorm 0.10 54,302 tok/s peakVRAM 13.7GB +[22:29:37] step 3160/4768 loss 3.0747 lr 6.00e-04 gnorm 0.09 54,304 tok/s peakVRAM 13.7GB +[22:32:50] step 3180/4768 loss 3.0803 lr 6.00e-04 gnorm 0.09 54,303 tok/s peakVRAM 13.7GB +[22:36:03] step 3200/4768 loss 3.0820 lr 6.00e-04 gnorm 0.09 54,306 tok/s peakVRAM 13.7GB +[22:39:16] step 3220/4768 loss 3.0916 lr 6.00e-04 gnorm 0.09 54,302 tok/s peakVRAM 13.7GB +[22:42:29] step 3240/4768 loss 3.0735 lr 6.00e-04 gnorm 0.09 54,302 tok/s peakVRAM 13.7GB +[22:45:42] step 3260/4768 loss 3.0701 lr 6.00e-04 gnorm 0.09 54,303 tok/s peakVRAM 13.7GB +[22:48:56] step 3280/4768 loss 3.0658 lr 6.00e-04 gnorm 0.09 54,301 tok/s peakVRAM 13.7GB +[22:52:09] step 3300/4768 loss 3.0763 lr 6.00e-04 gnorm 0.08 54,302 tok/s peakVRAM 13.7GB +[22:55:22] step 3320/4768 loss 3.0704 lr 6.00e-04 gnorm 0.09 54,303 tok/s peakVRAM 13.7GB +[22:58:35] step 3340/4768 loss 3.0707 lr 6.00e-04 gnorm 0.09 54,305 tok/s peakVRAM 13.7GB +[23:01:48] step 3360/4768 loss 3.0617 lr 6.00e-04 gnorm 0.10 54,304 tok/s peakVRAM 13.7GB +[23:05:01] step 3380/4768 loss 3.0715 lr 6.00e-04 gnorm 0.10 54,304 tok/s peakVRAM 13.7GB +[23:08:14] step 3400/4768 loss 3.0564 lr 6.00e-04 gnorm 0.09 54,306 tok/s peakVRAM 13.7GB +[23:11:27] step 3420/4768 loss 3.0647 lr 6.00e-04 gnorm 0.09 54,303 tok/s peakVRAM 13.7GB +[23:14:40] step 3440/4768 loss 3.0560 lr 6.00e-04 gnorm 0.10 54,306 tok/s peakVRAM 13.7GB +[23:17:53] step 3460/4768 loss 3.0657 lr 6.00e-04 gnorm 0.09 54,307 tok/s peakVRAM 13.7GB +[23:21:06] step 3480/4768 loss 3.0652 lr 6.00e-04 gnorm 0.09 54,304 tok/s peakVRAM 13.7GB +[23:24:20] step 3500/4768 loss 3.0513 lr 6.00e-04 gnorm 0.10 54,308 tok/s peakVRAM 13.7GB +[23:27:33] step 3520/4768 loss 3.0545 lr 6.00e-04 gnorm 0.10 54,304 tok/s peakVRAM 13.7GB +[23:30:46] step 3540/4768 loss 3.0561 lr 6.00e-04 gnorm 0.09 54,305 tok/s peakVRAM 13.7GB +[23:33:59] step 3560/4768 loss 3.0606 lr 6.00e-04 gnorm 0.09 54,303 tok/s peakVRAM 13.7GB +[23:37:12] step 3580/4768 loss 3.0544 lr 6.00e-04 gnorm 0.08 54,307 tok/s peakVRAM 13.7GB +[23:40:25] step 3600/4768 loss 3.0568 lr 6.00e-04 gnorm 0.10 54,309 tok/s peakVRAM 13.7GB +[23:43:38] step 3620/4768 loss 3.0479 lr 6.00e-04 gnorm 0.10 54,307 tok/s peakVRAM 13.7GB +[23:46:51] step 3640/4768 loss 3.0562 lr 6.00e-04 gnorm 0.09 54,308 tok/s peakVRAM 13.7GB +[23:50:04] step 3660/4768 loss 3.0452 lr 6.00e-04 gnorm 0.09 54,304 tok/s peakVRAM 13.7GB +[23:53:17] step 3680/4768 loss 3.0520 lr 6.00e-04 gnorm 0.09 54,305 tok/s peakVRAM 13.7GB +[23:56:30] step 3700/4768 loss 3.0491 lr 6.00e-04 gnorm 0.10 54,306 tok/s peakVRAM 13.7GB +[23:59:44] step 3720/4768 loss 3.0422 lr 6.00e-04 gnorm 0.10 54,305 tok/s peakVRAM 13.7GB +[00:02:57] step 3740/4768 loss 3.0386 lr 6.00e-04 gnorm 0.09 54,297 tok/s peakVRAM 13.7GB +[00:06:10] step 3760/4768 loss 3.0374 lr 6.00e-04 gnorm 0.10 54,301 tok/s peakVRAM 13.7GB +[00:09:23] step 3780/4768 loss 3.0376 lr 6.00e-04 gnorm 0.09 54,300 tok/s peakVRAM 13.7GB +[00:12:36] step 3800/4768 loss 3.0388 lr 6.00e-04 gnorm 0.09 54,301 tok/s peakVRAM 13.7GB +[00:15:49] step 3820/4768 loss 3.0327 lr 5.61e-04 gnorm 0.09 54,302 tok/s peakVRAM 13.7GB +[00:19:02] step 3840/4768 loss 3.0306 lr 5.13e-04 gnorm 0.08 54,308 tok/s peakVRAM 13.7GB +[00:22:15] step 3860/4768 loss 3.0235 lr 4.83e-04 gnorm 0.07 54,307 tok/s peakVRAM 13.7GB +[00:25:28] step 3880/4768 loss 3.0207 lr 4.59e-04 gnorm 0.07 54,307 tok/s peakVRAM 13.7GB +[00:28:41] step 3900/4768 loss 3.0140 lr 4.39e-04 gnorm 0.08 54,308 tok/s peakVRAM 13.7GB +[00:31:54] step 3920/4768 loss 3.0089 lr 4.21e-04 gnorm 0.08 54,310 tok/s peakVRAM 13.7GB +[00:35:08] step 3940/4768 loss 3.0008 lr 4.05e-04 gnorm 0.07 54,312 tok/s peakVRAM 13.7GB +[00:38:21] step 3960/4768 loss 3.0012 lr 3.89e-04 gnorm 0.08 54,309 tok/s peakVRAM 13.7GB +[00:41:34] step 3980/4768 loss 2.9935 lr 3.75e-04 gnorm 0.07 54,310 tok/s peakVRAM 13.7GB +[00:44:47] step 4000/4768 loss 2.9921 lr 3.62e-04 gnorm 0.07 54,306 tok/s peakVRAM 13.7GB +[00:48:00] step 4020/4768 loss 2.9892 lr 3.50e-04 gnorm 0.07 54,309 tok/s peakVRAM 13.7GB +[00:51:13] step 4040/4768 loss 2.9804 lr 3.38e-04 gnorm 0.07 54,307 tok/s peakVRAM 13.7GB +[00:54:26] step 4060/4768 loss 2.9800 lr 3.26e-04 gnorm 0.07 54,307 tok/s peakVRAM 13.7GB +[00:57:39] step 4080/4768 loss 2.9711 lr 3.15e-04 gnorm 0.07 54,306 tok/s peakVRAM 13.7GB +[01:00:52] step 4100/4768 loss 2.9771 lr 3.05e-04 gnorm 0.07 54,300 tok/s peakVRAM 13.7GB +[01:04:05] step 4120/4768 loss 2.9652 lr 2.95e-04 gnorm 0.07 54,296 tok/s peakVRAM 13.7GB +[01:07:18] step 4140/4768 loss 2.9576 lr 2.85e-04 gnorm 0.07 54,296 tok/s peakVRAM 13.7GB +[01:10:32] step 4160/4768 loss 2.9754 lr 2.75e-04 gnorm 0.07 54,296 tok/s peakVRAM 13.7GB +[01:13:45] step 4180/4768 loss 2.9682 lr 2.66e-04 gnorm 0.07 54,299 tok/s peakVRAM 13.7GB +[01:16:58] step 4200/4768 loss 2.9618 lr 2.57e-04 gnorm 0.07 54,299 tok/s peakVRAM 13.7GB +[01:20:11] step 4220/4768 loss 2.9553 lr 2.48e-04 gnorm 0.07 54,302 tok/s peakVRAM 13.7GB +[01:23:24] step 4240/4768 loss 2.9614 lr 2.40e-04 gnorm 0.07 54,297 tok/s peakVRAM 13.7GB +[01:26:37] step 4260/4768 loss 2.9540 lr 2.31e-04 gnorm 0.07 54,297 tok/s peakVRAM 13.7GB +[01:29:50] step 4280/4768 loss 2.9529 lr 2.23e-04 gnorm 0.07 54,296 tok/s peakVRAM 13.7GB +[01:33:03] step 4300/4768 loss 2.9506 lr 2.15e-04 gnorm 0.07 54,296 tok/s peakVRAM 13.7GB +[01:36:16] step 4320/4768 loss 2.9459 lr 2.07e-04 gnorm 0.07 54,297 tok/s peakVRAM 13.7GB +[01:39:30] step 4340/4768 loss 2.9443 lr 1.99e-04 gnorm 0.07 54,300 tok/s peakVRAM 13.7GB +[01:42:43] step 4360/4768 loss 2.9391 lr 1.92e-04 gnorm 0.07 54,298 tok/s peakVRAM 13.7GB +[01:45:56] step 4380/4768 loss 2.9453 lr 1.84e-04 gnorm 0.07 54,296 tok/s peakVRAM 13.7GB +[01:49:09] step 4400/4768 loss 2.9431 lr 1.77e-04 gnorm 0.07 54,300 tok/s peakVRAM 13.7GB +[01:52:22] step 4420/4768 loss 2.9360 lr 1.70e-04 gnorm 0.07 54,300 tok/s peakVRAM 13.7GB +[01:55:35] step 4440/4768 loss 2.9422 lr 1.63e-04 gnorm 0.07 54,300 tok/s peakVRAM 13.7GB +[01:58:48] step 4460/4768 loss 2.9381 lr 1.56e-04 gnorm 0.07 54,299 tok/s peakVRAM 13.7GB +[02:02:01] step 4480/4768 loss 2.9331 lr 1.49e-04 gnorm 0.07 54,298 tok/s peakVRAM 13.7GB +[02:05:15] step 4500/4768 loss 2.9303 lr 1.42e-04 gnorm 0.07 54,299 tok/s peakVRAM 13.7GB +[02:08:28] step 4520/4768 loss 2.9317 lr 1.36e-04 gnorm 0.07 54,299 tok/s peakVRAM 13.7GB +[02:11:41] step 4540/4768 loss 2.9233 lr 1.29e-04 gnorm 0.07 54,293 tok/s peakVRAM 13.7GB +[02:14:54] step 4560/4768 loss 2.9264 lr 1.23e-04 gnorm 0.07 54,295 tok/s peakVRAM 13.7GB +[02:18:07] step 4580/4768 loss 2.9300 lr 1.16e-04 gnorm 0.07 54,293 tok/s peakVRAM 13.7GB +[02:21:20] step 4600/4768 loss 2.9284 lr 1.10e-04 gnorm 0.07 54,295 tok/s peakVRAM 13.7GB +[02:24:33] step 4620/4768 loss 2.9155 lr 1.04e-04 gnorm 0.07 54,290 tok/s peakVRAM 13.7GB +[02:27:46] step 4640/4768 loss 2.9244 lr 9.78e-05 gnorm 0.07 54,291 tok/s peakVRAM 13.7GB +[02:31:00] step 4660/4768 loss 2.9173 lr 9.18e-05 gnorm 0.07 54,298 tok/s peakVRAM 13.7GB +[02:34:13] step 4680/4768 loss 2.9151 lr 8.58e-05 gnorm 0.07 54,299 tok/s peakVRAM 13.7GB +[02:37:26] step 4700/4768 loss 2.9109 lr 7.99e-05 gnorm 0.07 54,299 tok/s peakVRAM 13.7GB +[02:40:39] step 4720/4768 loss 2.9122 lr 7.41e-05 gnorm 0.07 54,298 tok/s peakVRAM 13.7GB +[02:43:52] step 4740/4768 loss 2.9100 lr 6.83e-05 gnorm 0.07 54,301 tok/s peakVRAM 13.7GB +[02:47:05] step 4760/4768 loss 2.9194 lr 6.26e-05 gnorm 0.07 54,313 tok/s peakVRAM 13.7GB +[02:48:24] ckpt @ 4768 +[02:48:24] DONE diff --git a/R3/run/train.out b/R3/run/train.out new file mode 100644 index 0000000000000000000000000000000000000000..0af7b4d9109bc1f909e505147706aa7c88143d64 --- /dev/null +++ b/R3/run/train.out @@ -0,0 +1,248 @@ +[13:59:56] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[13:59:56] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[13:59:58] model GPT-ref: 183.6M param (L16 d896 h14) +[13:59:59] optimizer=muon muon_lr=0.02 (mult=33.3x) +[13:59:59] r11 pamiec: act_ckpt=True loss_chunk=1024 +[13:59:59] r13: mixer_layout=AAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[14:00:00] torch.compile enabled +[14:00:00] START gpt-ref: steps=4768 (od 0) lr=0.0006->6e-05 +[14:04:22] step 20/4768 loss 10.3912 lr 2.40e-05 gnorm 2.73 39,938 tok/s peakVRAM 13.7GB +[14:07:35] step 40/4768 loss 9.5518 lr 4.80e-05 gnorm 1.36 54,379 tok/s peakVRAM 13.7GB +[14:10:48] step 60/4768 loss 8.6782 lr 7.20e-05 gnorm 0.73 54,389 tok/s peakVRAM 13.7GB +[14:14:00] step 80/4768 loss 8.0385 lr 9.60e-05 gnorm 1.30 54,384 tok/s peakVRAM 13.7GB +[14:17:13] step 100/4768 loss 7.5974 lr 1.20e-04 gnorm 0.84 54,364 tok/s peakVRAM 13.7GB +[14:20:26] step 120/4768 loss 7.1265 lr 1.44e-04 gnorm 0.78 54,354 tok/s peakVRAM 13.7GB +[14:23:39] step 140/4768 loss 6.7082 lr 1.68e-04 gnorm 0.88 54,350 tok/s peakVRAM 13.7GB +[14:26:52] step 160/4768 loss 6.3537 lr 1.92e-04 gnorm 0.78 54,345 tok/s peakVRAM 13.7GB +[14:30:05] step 180/4768 loss 6.0894 lr 2.16e-04 gnorm 0.86 54,335 tok/s peakVRAM 13.7GB +[14:33:18] step 200/4768 loss 5.8676 lr 2.40e-04 gnorm 0.76 54,330 tok/s peakVRAM 13.7GB +[14:36:31] step 220/4768 loss 5.6845 lr 2.64e-04 gnorm 0.84 54,330 tok/s peakVRAM 13.7GB +[14:39:44] step 240/4768 loss 5.5133 lr 2.88e-04 gnorm 0.76 54,330 tok/s peakVRAM 13.7GB +[14:42:57] step 260/4768 loss 5.3498 lr 3.12e-04 gnorm 0.70 54,329 tok/s peakVRAM 13.7GB +[14:46:10] step 280/4768 loss 5.1918 lr 3.36e-04 gnorm 0.66 54,330 tok/s peakVRAM 13.7GB +[14:49:23] step 300/4768 loss 5.0466 lr 3.60e-04 gnorm 0.66 54,328 tok/s peakVRAM 13.7GB +[14:52:36] step 320/4768 loss 4.9124 lr 3.84e-04 gnorm 0.72 54,329 tok/s peakVRAM 13.7GB +[14:55:49] step 340/4768 loss 4.7663 lr 4.08e-04 gnorm 0.64 54,326 tok/s peakVRAM 13.7GB +[14:59:02] step 360/4768 loss 4.6379 lr 4.32e-04 gnorm 0.60 54,329 tok/s peakVRAM 13.7GB +[15:02:15] step 380/4768 loss 4.5207 lr 4.56e-04 gnorm 0.50 54,328 tok/s peakVRAM 13.7GB +[15:05:28] step 400/4768 loss 4.4147 lr 4.80e-04 gnorm 0.44 54,326 tok/s peakVRAM 13.7GB +[15:08:41] step 420/4768 loss 4.3344 lr 5.04e-04 gnorm 0.48 54,325 tok/s peakVRAM 13.7GB +[15:11:54] step 440/4768 loss 4.2526 lr 5.28e-04 gnorm 0.36 54,324 tok/s peakVRAM 13.7GB +[15:15:07] step 460/4768 loss 4.1822 lr 5.52e-04 gnorm 0.47 54,322 tok/s peakVRAM 13.7GB +[15:18:20] step 480/4768 loss 4.1330 lr 5.76e-04 gnorm 0.38 54,320 tok/s peakVRAM 13.7GB +[15:21:33] step 500/4768 loss 4.0766 lr 6.00e-04 gnorm 0.30 54,320 tok/s peakVRAM 13.7GB +[15:24:46] step 520/4768 loss 4.0389 lr 6.00e-04 gnorm 0.35 54,316 tok/s peakVRAM 13.7GB +[15:27:59] step 540/4768 loss 3.9892 lr 6.00e-04 gnorm 0.26 54,316 tok/s peakVRAM 13.7GB +[15:31:13] step 560/4768 loss 3.9361 lr 6.00e-04 gnorm 0.24 54,314 tok/s peakVRAM 13.7GB +[15:34:26] step 580/4768 loss 3.8941 lr 6.00e-04 gnorm 0.24 54,311 tok/s peakVRAM 13.7GB +[15:37:39] step 600/4768 loss 3.8519 lr 6.00e-04 gnorm 0.22 54,312 tok/s peakVRAM 13.7GB +[15:40:52] step 620/4768 loss 3.8298 lr 6.00e-04 gnorm 0.22 54,314 tok/s peakVRAM 13.7GB +[15:44:05] step 640/4768 loss 3.8032 lr 6.00e-04 gnorm 0.19 54,308 tok/s peakVRAM 13.7GB +[15:47:18] step 660/4768 loss 3.7710 lr 6.00e-04 gnorm 0.18 54,309 tok/s peakVRAM 13.7GB +[15:50:31] step 680/4768 loss 3.7483 lr 6.00e-04 gnorm 0.18 54,309 tok/s peakVRAM 13.7GB +[15:53:44] step 700/4768 loss 3.7176 lr 6.00e-04 gnorm 0.17 54,304 tok/s peakVRAM 13.7GB +[15:56:57] step 720/4768 loss 3.6874 lr 6.00e-04 gnorm 0.18 54,296 tok/s peakVRAM 13.7GB +[16:00:10] step 740/4768 loss 3.6829 lr 6.00e-04 gnorm 0.18 54,301 tok/s peakVRAM 13.7GB +[16:03:23] step 760/4768 loss 3.6580 lr 6.00e-04 gnorm 0.15 54,298 tok/s peakVRAM 13.7GB +[16:06:36] step 780/4768 loss 3.6315 lr 6.00e-04 gnorm 0.16 54,301 tok/s peakVRAM 13.7GB +[16:09:50] step 800/4768 loss 3.6186 lr 6.00e-04 gnorm 0.17 54,303 tok/s peakVRAM 13.7GB +[16:13:03] step 820/4768 loss 3.6022 lr 6.00e-04 gnorm 0.15 54,305 tok/s peakVRAM 13.7GB +[16:16:16] step 840/4768 loss 3.5874 lr 6.00e-04 gnorm 0.17 54,300 tok/s peakVRAM 13.7GB +[16:19:29] step 860/4768 loss 3.5672 lr 6.00e-04 gnorm 0.16 54,296 tok/s peakVRAM 13.7GB +[16:22:42] step 880/4768 loss 3.5612 lr 6.00e-04 gnorm 0.13 54,301 tok/s peakVRAM 13.7GB +[16:25:55] step 900/4768 loss 3.5428 lr 6.00e-04 gnorm 0.14 54,306 tok/s peakVRAM 13.7GB +[16:29:08] step 920/4768 loss 3.5357 lr 6.00e-04 gnorm 0.14 54,304 tok/s peakVRAM 13.7GB +[16:32:21] step 940/4768 loss 3.5316 lr 6.00e-04 gnorm 0.13 54,308 tok/s peakVRAM 13.7GB +[16:35:34] step 960/4768 loss 3.5057 lr 6.00e-04 gnorm 0.13 54,309 tok/s peakVRAM 13.7GB +[16:38:47] step 980/4768 loss 3.5031 lr 6.00e-04 gnorm 0.14 54,307 tok/s peakVRAM 13.7GB +[16:42:01] step 1000/4768 loss 3.4927 lr 6.00e-04 gnorm 0.13 54,302 tok/s peakVRAM 13.7GB +[16:45:14] step 1020/4768 loss 3.4872 lr 6.00e-04 gnorm 0.13 54,303 tok/s peakVRAM 13.7GB +[16:48:27] step 1040/4768 loss 3.4604 lr 6.00e-04 gnorm 0.13 54,306 tok/s peakVRAM 13.7GB +[16:51:40] step 1060/4768 loss 3.4721 lr 6.00e-04 gnorm 0.13 54,302 tok/s peakVRAM 13.7GB +[16:54:53] step 1080/4768 loss 3.4612 lr 6.00e-04 gnorm 0.13 54,303 tok/s peakVRAM 13.7GB +[16:58:06] step 1100/4768 loss 3.4462 lr 6.00e-04 gnorm 0.14 54,300 tok/s peakVRAM 13.7GB +[17:01:19] step 1120/4768 loss 3.4286 lr 6.00e-04 gnorm 0.13 54,299 tok/s peakVRAM 13.7GB +[17:04:32] step 1140/4768 loss 3.4225 lr 6.00e-04 gnorm 0.12 54,296 tok/s peakVRAM 13.7GB +[17:07:45] step 1160/4768 loss 3.4164 lr 6.00e-04 gnorm 0.12 54,303 tok/s peakVRAM 13.7GB +[17:10:58] step 1180/4768 loss 3.4192 lr 6.00e-04 gnorm 0.12 54,303 tok/s peakVRAM 13.7GB +[17:14:12] step 1200/4768 loss 3.4006 lr 6.00e-04 gnorm 0.12 54,301 tok/s peakVRAM 13.7GB +[17:17:25] step 1220/4768 loss 3.3984 lr 6.00e-04 gnorm 0.11 54,291 tok/s peakVRAM 13.7GB +[17:20:38] step 1240/4768 loss 3.3901 lr 6.00e-04 gnorm 0.12 54,296 tok/s peakVRAM 13.7GB +[17:23:51] step 1260/4768 loss 3.3763 lr 6.00e-04 gnorm 0.13 54,303 tok/s peakVRAM 13.7GB +[17:27:04] step 1280/4768 loss 3.3792 lr 6.00e-04 gnorm 0.12 54,301 tok/s peakVRAM 13.7GB +[17:30:17] step 1300/4768 loss 3.3721 lr 6.00e-04 gnorm 0.13 54,297 tok/s peakVRAM 13.7GB +[17:33:30] step 1320/4768 loss 3.3646 lr 6.00e-04 gnorm 0.11 54,295 tok/s peakVRAM 13.7GB +[17:36:43] step 1340/4768 loss 3.3652 lr 6.00e-04 gnorm 0.13 54,297 tok/s peakVRAM 13.7GB +[17:39:57] step 1360/4768 loss 3.3561 lr 6.00e-04 gnorm 0.12 54,297 tok/s peakVRAM 13.7GB +[17:43:10] step 1380/4768 loss 3.3482 lr 6.00e-04 gnorm 0.13 54,301 tok/s peakVRAM 13.7GB +[17:46:23] step 1400/4768 loss 3.3375 lr 6.00e-04 gnorm 0.12 54,298 tok/s peakVRAM 13.7GB +[17:49:36] step 1420/4768 loss 3.3256 lr 6.00e-04 gnorm 0.11 54,301 tok/s peakVRAM 13.7GB +[17:52:49] step 1440/4768 loss 3.3326 lr 6.00e-04 gnorm 0.12 54,303 tok/s peakVRAM 13.7GB +[17:56:02] step 1460/4768 loss 3.3301 lr 6.00e-04 gnorm 0.12 54,296 tok/s peakVRAM 13.7GB +[17:59:15] step 1480/4768 loss 3.3341 lr 6.00e-04 gnorm 0.11 54,299 tok/s peakVRAM 13.7GB +[18:02:28] step 1500/4768 loss 3.3077 lr 6.00e-04 gnorm 0.12 54,299 tok/s peakVRAM 13.7GB +[18:05:41] step 1520/4768 loss 3.3145 lr 6.00e-04 gnorm 0.11 54,297 tok/s peakVRAM 13.7GB +[18:08:55] step 1540/4768 loss 3.2987 lr 6.00e-04 gnorm 0.11 54,295 tok/s peakVRAM 13.7GB +[18:12:08] step 1560/4768 loss 3.3075 lr 6.00e-04 gnorm 0.12 54,296 tok/s peakVRAM 13.7GB +[18:15:21] step 1580/4768 loss 3.2953 lr 6.00e-04 gnorm 0.12 54,292 tok/s peakVRAM 13.7GB +[18:18:34] step 1600/4768 loss 3.2850 lr 6.00e-04 gnorm 0.11 54,290 tok/s peakVRAM 13.7GB +[18:21:47] step 1620/4768 loss 3.2859 lr 6.00e-04 gnorm 0.11 54,296 tok/s peakVRAM 13.7GB +[18:25:00] step 1640/4768 loss 3.2823 lr 6.00e-04 gnorm 0.11 54,294 tok/s peakVRAM 13.7GB +[18:28:13] step 1660/4768 loss 3.2832 lr 6.00e-04 gnorm 0.11 54,295 tok/s peakVRAM 13.7GB +[18:31:26] step 1680/4768 loss 3.2724 lr 6.00e-04 gnorm 0.11 54,296 tok/s peakVRAM 13.7GB +[18:34:40] step 1700/4768 loss 3.2652 lr 6.00e-04 gnorm 0.12 54,297 tok/s peakVRAM 13.7GB +[18:37:53] step 1720/4768 loss 3.2673 lr 6.00e-04 gnorm 0.11 54,291 tok/s peakVRAM 13.7GB +[18:41:06] step 1740/4768 loss 3.2613 lr 6.00e-04 gnorm 0.10 54,301 tok/s peakVRAM 13.7GB +[18:44:19] step 1760/4768 loss 3.2605 lr 6.00e-04 gnorm 0.11 54,297 tok/s peakVRAM 13.7GB +[18:47:32] step 1780/4768 loss 3.2474 lr 6.00e-04 gnorm 0.10 54,297 tok/s peakVRAM 13.7GB +[18:50:45] step 1800/4768 loss 3.2512 lr 6.00e-04 gnorm 0.11 54,296 tok/s peakVRAM 13.7GB +[18:53:58] step 1820/4768 loss 3.2471 lr 6.00e-04 gnorm 0.11 54,297 tok/s peakVRAM 13.7GB +[18:57:11] step 1840/4768 loss 3.2405 lr 6.00e-04 gnorm 0.11 54,297 tok/s peakVRAM 13.7GB +[19:00:25] step 1860/4768 loss 3.2397 lr 6.00e-04 gnorm 0.11 54,291 tok/s peakVRAM 13.7GB +[19:03:38] step 1880/4768 loss 3.2355 lr 6.00e-04 gnorm 0.10 54,290 tok/s peakVRAM 13.7GB +[19:06:51] step 1900/4768 loss 3.2295 lr 6.00e-04 gnorm 0.11 54,292 tok/s peakVRAM 13.7GB +[19:10:04] step 1920/4768 loss 3.2302 lr 6.00e-04 gnorm 0.11 54,291 tok/s peakVRAM 13.7GB +[19:13:17] step 1940/4768 loss 3.2224 lr 6.00e-04 gnorm 0.10 54,290 tok/s peakVRAM 13.7GB +[19:16:30] step 1960/4768 loss 3.2266 lr 6.00e-04 gnorm 0.11 54,292 tok/s peakVRAM 13.7GB +[19:19:43] step 1980/4768 loss 3.2208 lr 6.00e-04 gnorm 0.09 54,293 tok/s peakVRAM 13.7GB +[19:22:56] step 2000/4768 loss 3.2143 lr 6.00e-04 gnorm 0.12 54,302 tok/s peakVRAM 13.7GB +[19:26:10] step 2020/4768 loss 3.2143 lr 6.00e-04 gnorm 0.11 54,303 tok/s peakVRAM 13.7GB +[19:29:23] step 2040/4768 loss 3.2074 lr 6.00e-04 gnorm 0.12 54,303 tok/s peakVRAM 13.7GB +[19:32:36] step 2060/4768 loss 3.2075 lr 6.00e-04 gnorm 0.10 54,298 tok/s peakVRAM 13.7GB +[19:35:49] step 2080/4768 loss 3.1964 lr 6.00e-04 gnorm 0.09 54,300 tok/s peakVRAM 13.7GB +[19:39:02] step 2100/4768 loss 3.2047 lr 6.00e-04 gnorm 0.10 54,299 tok/s peakVRAM 13.7GB +[19:42:15] step 2120/4768 loss 3.1944 lr 6.00e-04 gnorm 0.11 54,296 tok/s peakVRAM 13.7GB +[19:45:28] step 2140/4768 loss 3.2072 lr 6.00e-04 gnorm 0.11 54,298 tok/s peakVRAM 13.7GB +[19:48:41] step 2160/4768 loss 3.1865 lr 6.00e-04 gnorm 0.10 54,300 tok/s peakVRAM 13.7GB +[19:51:54] step 2180/4768 loss 3.1878 lr 6.00e-04 gnorm 0.10 54,301 tok/s peakVRAM 13.7GB +[19:55:08] step 2200/4768 loss 3.1888 lr 6.00e-04 gnorm 0.11 54,297 tok/s peakVRAM 13.7GB +[19:58:21] step 2220/4768 loss 3.1810 lr 6.00e-04 gnorm 0.10 54,295 tok/s peakVRAM 13.7GB +[20:01:34] step 2240/4768 loss 3.1782 lr 6.00e-04 gnorm 0.11 54,293 tok/s peakVRAM 13.7GB +[20:04:47] step 2260/4768 loss 3.1746 lr 6.00e-04 gnorm 0.11 54,296 tok/s peakVRAM 13.7GB +[20:08:00] step 2280/4768 loss 3.1712 lr 6.00e-04 gnorm 0.11 54,291 tok/s peakVRAM 13.7GB +[20:11:13] step 2300/4768 loss 3.1743 lr 6.00e-04 gnorm 0.11 54,296 tok/s peakVRAM 13.7GB +[20:14:26] step 2320/4768 loss 3.1707 lr 6.00e-04 gnorm 0.10 54,297 tok/s peakVRAM 13.7GB +[20:17:39] step 2340/4768 loss 3.1656 lr 6.00e-04 gnorm 0.10 54,299 tok/s peakVRAM 13.7GB +[20:20:53] step 2360/4768 loss 3.1564 lr 6.00e-04 gnorm 0.10 54,294 tok/s peakVRAM 13.7GB +[20:24:06] step 2380/4768 loss 3.1555 lr 6.00e-04 gnorm 0.10 54,296 tok/s peakVRAM 13.7GB +[20:27:19] step 2400/4768 loss 3.1551 lr 6.00e-04 gnorm 0.10 54,296 tok/s peakVRAM 13.7GB +[20:30:32] step 2420/4768 loss 3.1583 lr 6.00e-04 gnorm 0.10 54,299 tok/s peakVRAM 13.7GB +[20:33:45] step 2440/4768 loss 3.1520 lr 6.00e-04 gnorm 0.10 54,302 tok/s peakVRAM 13.7GB +[20:36:58] step 2460/4768 loss 3.1510 lr 6.00e-04 gnorm 0.11 54,303 tok/s peakVRAM 13.7GB +[20:40:11] step 2480/4768 loss 3.1482 lr 6.00e-04 gnorm 0.10 54,302 tok/s peakVRAM 13.7GB +[20:43:24] step 2500/4768 loss 3.1426 lr 6.00e-04 gnorm 0.10 54,300 tok/s peakVRAM 13.7GB +[20:46:37] step 2520/4768 loss 3.1444 lr 6.00e-04 gnorm 0.10 54,302 tok/s peakVRAM 13.7GB +[20:49:51] step 2540/4768 loss 3.1448 lr 6.00e-04 gnorm 0.11 54,301 tok/s peakVRAM 13.7GB +[20:53:04] step 2560/4768 loss 3.1416 lr 6.00e-04 gnorm 0.09 54,302 tok/s peakVRAM 13.7GB +[20:56:17] step 2580/4768 loss 3.1369 lr 6.00e-04 gnorm 0.09 54,297 tok/s peakVRAM 13.7GB +[20:59:30] step 2600/4768 loss 3.1276 lr 6.00e-04 gnorm 0.10 54,298 tok/s peakVRAM 13.7GB +[21:02:43] step 2620/4768 loss 3.1362 lr 6.00e-04 gnorm 0.10 54,301 tok/s peakVRAM 13.7GB +[21:05:56] step 2640/4768 loss 3.1374 lr 6.00e-04 gnorm 0.10 54,303 tok/s peakVRAM 13.7GB +[21:09:09] step 2660/4768 loss 3.1362 lr 6.00e-04 gnorm 0.11 54,301 tok/s peakVRAM 13.7GB +[21:12:22] step 2680/4768 loss 3.1310 lr 6.00e-04 gnorm 0.11 54,301 tok/s peakVRAM 13.7GB +[21:15:35] step 2700/4768 loss 3.1338 lr 6.00e-04 gnorm 0.09 54,299 tok/s peakVRAM 13.7GB +[21:18:48] step 2720/4768 loss 3.1165 lr 6.00e-04 gnorm 0.10 54,302 tok/s peakVRAM 13.7GB +[21:22:02] step 2740/4768 loss 3.1183 lr 6.00e-04 gnorm 0.10 54,302 tok/s peakVRAM 13.7GB +[21:25:15] step 2760/4768 loss 3.1214 lr 6.00e-04 gnorm 0.10 54,301 tok/s peakVRAM 13.7GB +[21:28:28] step 2780/4768 loss 3.1122 lr 6.00e-04 gnorm 0.10 54,299 tok/s peakVRAM 13.7GB +[21:31:41] step 2800/4768 loss 3.1071 lr 6.00e-04 gnorm 0.11 54,294 tok/s peakVRAM 13.7GB +[21:34:54] step 2820/4768 loss 3.1044 lr 6.00e-04 gnorm 0.09 54,294 tok/s peakVRAM 13.7GB +[21:38:07] step 2840/4768 loss 3.1124 lr 6.00e-04 gnorm 0.10 54,299 tok/s peakVRAM 13.7GB +[21:41:20] step 2860/4768 loss 3.1040 lr 6.00e-04 gnorm 0.09 54,296 tok/s peakVRAM 13.7GB +[21:44:33] step 2880/4768 loss 3.1058 lr 6.00e-04 gnorm 0.10 54,297 tok/s peakVRAM 13.7GB +[21:47:47] step 2900/4768 loss 3.1077 lr 6.00e-04 gnorm 0.10 54,301 tok/s peakVRAM 13.7GB +[21:51:00] step 2920/4768 loss 3.1145 lr 6.00e-04 gnorm 0.09 54,298 tok/s peakVRAM 13.7GB +[21:54:13] step 2940/4768 loss 3.1047 lr 6.00e-04 gnorm 0.08 54,296 tok/s peakVRAM 13.7GB +[21:57:26] step 2960/4768 loss 3.1066 lr 6.00e-04 gnorm 0.10 54,297 tok/s peakVRAM 13.7GB +[22:00:39] step 2980/4768 loss 3.0944 lr 6.00e-04 gnorm 0.11 54,300 tok/s peakVRAM 13.7GB +[22:03:52] step 3000/4768 loss 3.0945 lr 6.00e-04 gnorm 0.09 54,295 tok/s peakVRAM 13.7GB +[22:07:05] step 3020/4768 loss 3.0901 lr 6.00e-04 gnorm 0.09 54,296 tok/s peakVRAM 13.7GB +[22:10:18] step 3040/4768 loss 3.0874 lr 6.00e-04 gnorm 0.10 54,295 tok/s peakVRAM 13.7GB +[22:13:31] step 3060/4768 loss 3.0911 lr 6.00e-04 gnorm 0.10 54,304 tok/s peakVRAM 13.7GB +[22:16:45] step 3080/4768 loss 3.0834 lr 6.00e-04 gnorm 0.09 54,304 tok/s peakVRAM 13.7GB +[22:19:58] step 3100/4768 loss 3.0775 lr 6.00e-04 gnorm 0.10 54,303 tok/s peakVRAM 13.7GB +[22:23:11] step 3120/4768 loss 3.0931 lr 6.00e-04 gnorm 0.10 54,302 tok/s peakVRAM 13.7GB +[22:26:24] step 3140/4768 loss 3.0785 lr 6.00e-04 gnorm 0.10 54,302 tok/s peakVRAM 13.7GB +[22:29:37] step 3160/4768 loss 3.0747 lr 6.00e-04 gnorm 0.09 54,304 tok/s peakVRAM 13.7GB +[22:32:50] step 3180/4768 loss 3.0803 lr 6.00e-04 gnorm 0.09 54,303 tok/s peakVRAM 13.7GB +[22:36:03] step 3200/4768 loss 3.0820 lr 6.00e-04 gnorm 0.09 54,306 tok/s peakVRAM 13.7GB +[22:39:16] step 3220/4768 loss 3.0916 lr 6.00e-04 gnorm 0.09 54,302 tok/s peakVRAM 13.7GB +[22:42:29] step 3240/4768 loss 3.0735 lr 6.00e-04 gnorm 0.09 54,302 tok/s peakVRAM 13.7GB +[22:45:42] step 3260/4768 loss 3.0701 lr 6.00e-04 gnorm 0.09 54,303 tok/s peakVRAM 13.7GB +[22:48:56] step 3280/4768 loss 3.0658 lr 6.00e-04 gnorm 0.09 54,301 tok/s peakVRAM 13.7GB +[22:52:09] step 3300/4768 loss 3.0763 lr 6.00e-04 gnorm 0.08 54,302 tok/s peakVRAM 13.7GB +[22:55:22] step 3320/4768 loss 3.0704 lr 6.00e-04 gnorm 0.09 54,303 tok/s peakVRAM 13.7GB +[22:58:35] step 3340/4768 loss 3.0707 lr 6.00e-04 gnorm 0.09 54,305 tok/s peakVRAM 13.7GB +[23:01:48] step 3360/4768 loss 3.0617 lr 6.00e-04 gnorm 0.10 54,304 tok/s peakVRAM 13.7GB +[23:05:01] step 3380/4768 loss 3.0715 lr 6.00e-04 gnorm 0.10 54,304 tok/s peakVRAM 13.7GB +[23:08:14] step 3400/4768 loss 3.0564 lr 6.00e-04 gnorm 0.09 54,306 tok/s peakVRAM 13.7GB +[23:11:27] step 3420/4768 loss 3.0647 lr 6.00e-04 gnorm 0.09 54,303 tok/s peakVRAM 13.7GB +[23:14:40] step 3440/4768 loss 3.0560 lr 6.00e-04 gnorm 0.10 54,306 tok/s peakVRAM 13.7GB +[23:17:53] step 3460/4768 loss 3.0657 lr 6.00e-04 gnorm 0.09 54,307 tok/s peakVRAM 13.7GB +[23:21:06] step 3480/4768 loss 3.0652 lr 6.00e-04 gnorm 0.09 54,304 tok/s peakVRAM 13.7GB +[23:24:20] step 3500/4768 loss 3.0513 lr 6.00e-04 gnorm 0.10 54,308 tok/s peakVRAM 13.7GB +[23:27:33] step 3520/4768 loss 3.0545 lr 6.00e-04 gnorm 0.10 54,304 tok/s peakVRAM 13.7GB +[23:30:46] step 3540/4768 loss 3.0561 lr 6.00e-04 gnorm 0.09 54,305 tok/s peakVRAM 13.7GB +[23:33:59] step 3560/4768 loss 3.0606 lr 6.00e-04 gnorm 0.09 54,303 tok/s peakVRAM 13.7GB +[23:37:12] step 3580/4768 loss 3.0544 lr 6.00e-04 gnorm 0.08 54,307 tok/s peakVRAM 13.7GB +[23:40:25] step 3600/4768 loss 3.0568 lr 6.00e-04 gnorm 0.10 54,309 tok/s peakVRAM 13.7GB +[23:43:38] step 3620/4768 loss 3.0479 lr 6.00e-04 gnorm 0.10 54,307 tok/s peakVRAM 13.7GB +[23:46:51] step 3640/4768 loss 3.0562 lr 6.00e-04 gnorm 0.09 54,308 tok/s peakVRAM 13.7GB +[23:50:04] step 3660/4768 loss 3.0452 lr 6.00e-04 gnorm 0.09 54,304 tok/s peakVRAM 13.7GB +[23:53:17] step 3680/4768 loss 3.0520 lr 6.00e-04 gnorm 0.09 54,305 tok/s peakVRAM 13.7GB +[23:56:30] step 3700/4768 loss 3.0491 lr 6.00e-04 gnorm 0.10 54,306 tok/s peakVRAM 13.7GB +[23:59:44] step 3720/4768 loss 3.0422 lr 6.00e-04 gnorm 0.10 54,305 tok/s peakVRAM 13.7GB +[00:02:57] step 3740/4768 loss 3.0386 lr 6.00e-04 gnorm 0.09 54,297 tok/s peakVRAM 13.7GB +[00:06:10] step 3760/4768 loss 3.0374 lr 6.00e-04 gnorm 0.10 54,301 tok/s peakVRAM 13.7GB +[00:09:23] step 3780/4768 loss 3.0376 lr 6.00e-04 gnorm 0.09 54,300 tok/s peakVRAM 13.7GB +[00:12:36] step 3800/4768 loss 3.0388 lr 6.00e-04 gnorm 0.09 54,301 tok/s peakVRAM 13.7GB +[00:15:49] step 3820/4768 loss 3.0327 lr 5.61e-04 gnorm 0.09 54,302 tok/s peakVRAM 13.7GB +[00:19:02] step 3840/4768 loss 3.0306 lr 5.13e-04 gnorm 0.08 54,308 tok/s peakVRAM 13.7GB +[00:22:15] step 3860/4768 loss 3.0235 lr 4.83e-04 gnorm 0.07 54,307 tok/s peakVRAM 13.7GB +[00:25:28] step 3880/4768 loss 3.0207 lr 4.59e-04 gnorm 0.07 54,307 tok/s peakVRAM 13.7GB +[00:28:41] step 3900/4768 loss 3.0140 lr 4.39e-04 gnorm 0.08 54,308 tok/s peakVRAM 13.7GB +[00:31:54] step 3920/4768 loss 3.0089 lr 4.21e-04 gnorm 0.08 54,310 tok/s peakVRAM 13.7GB +[00:35:08] step 3940/4768 loss 3.0008 lr 4.05e-04 gnorm 0.07 54,312 tok/s peakVRAM 13.7GB +[00:38:21] step 3960/4768 loss 3.0012 lr 3.89e-04 gnorm 0.08 54,309 tok/s peakVRAM 13.7GB +[00:41:34] step 3980/4768 loss 2.9935 lr 3.75e-04 gnorm 0.07 54,310 tok/s peakVRAM 13.7GB +[00:44:47] step 4000/4768 loss 2.9921 lr 3.62e-04 gnorm 0.07 54,306 tok/s peakVRAM 13.7GB +[00:48:00] step 4020/4768 loss 2.9892 lr 3.50e-04 gnorm 0.07 54,309 tok/s peakVRAM 13.7GB +[00:51:13] step 4040/4768 loss 2.9804 lr 3.38e-04 gnorm 0.07 54,307 tok/s peakVRAM 13.7GB +[00:54:26] step 4060/4768 loss 2.9800 lr 3.26e-04 gnorm 0.07 54,307 tok/s peakVRAM 13.7GB +[00:57:39] step 4080/4768 loss 2.9711 lr 3.15e-04 gnorm 0.07 54,306 tok/s peakVRAM 13.7GB +[01:00:52] step 4100/4768 loss 2.9771 lr 3.05e-04 gnorm 0.07 54,300 tok/s peakVRAM 13.7GB +[01:04:05] step 4120/4768 loss 2.9652 lr 2.95e-04 gnorm 0.07 54,296 tok/s peakVRAM 13.7GB +[01:07:18] step 4140/4768 loss 2.9576 lr 2.85e-04 gnorm 0.07 54,296 tok/s peakVRAM 13.7GB +[01:10:32] step 4160/4768 loss 2.9754 lr 2.75e-04 gnorm 0.07 54,296 tok/s peakVRAM 13.7GB +[01:13:45] step 4180/4768 loss 2.9682 lr 2.66e-04 gnorm 0.07 54,299 tok/s peakVRAM 13.7GB +[01:16:58] step 4200/4768 loss 2.9618 lr 2.57e-04 gnorm 0.07 54,299 tok/s peakVRAM 13.7GB +[01:20:11] step 4220/4768 loss 2.9553 lr 2.48e-04 gnorm 0.07 54,302 tok/s peakVRAM 13.7GB +[01:23:24] step 4240/4768 loss 2.9614 lr 2.40e-04 gnorm 0.07 54,297 tok/s peakVRAM 13.7GB +[01:26:37] step 4260/4768 loss 2.9540 lr 2.31e-04 gnorm 0.07 54,297 tok/s peakVRAM 13.7GB +[01:29:50] step 4280/4768 loss 2.9529 lr 2.23e-04 gnorm 0.07 54,296 tok/s peakVRAM 13.7GB +[01:33:03] step 4300/4768 loss 2.9506 lr 2.15e-04 gnorm 0.07 54,296 tok/s peakVRAM 13.7GB +[01:36:16] step 4320/4768 loss 2.9459 lr 2.07e-04 gnorm 0.07 54,297 tok/s peakVRAM 13.7GB +[01:39:30] step 4340/4768 loss 2.9443 lr 1.99e-04 gnorm 0.07 54,300 tok/s peakVRAM 13.7GB +[01:42:43] step 4360/4768 loss 2.9391 lr 1.92e-04 gnorm 0.07 54,298 tok/s peakVRAM 13.7GB +[01:45:56] step 4380/4768 loss 2.9453 lr 1.84e-04 gnorm 0.07 54,296 tok/s peakVRAM 13.7GB +[01:49:09] step 4400/4768 loss 2.9431 lr 1.77e-04 gnorm 0.07 54,300 tok/s peakVRAM 13.7GB +[01:52:22] step 4420/4768 loss 2.9360 lr 1.70e-04 gnorm 0.07 54,300 tok/s peakVRAM 13.7GB +[01:55:35] step 4440/4768 loss 2.9422 lr 1.63e-04 gnorm 0.07 54,300 tok/s peakVRAM 13.7GB +[01:58:48] step 4460/4768 loss 2.9381 lr 1.56e-04 gnorm 0.07 54,299 tok/s peakVRAM 13.7GB +[02:02:01] step 4480/4768 loss 2.9331 lr 1.49e-04 gnorm 0.07 54,298 tok/s peakVRAM 13.7GB +[02:05:15] step 4500/4768 loss 2.9303 lr 1.42e-04 gnorm 0.07 54,299 tok/s peakVRAM 13.7GB +[02:08:28] step 4520/4768 loss 2.9317 lr 1.36e-04 gnorm 0.07 54,299 tok/s peakVRAM 13.7GB +[02:11:41] step 4540/4768 loss 2.9233 lr 1.29e-04 gnorm 0.07 54,293 tok/s peakVRAM 13.7GB +[02:14:54] step 4560/4768 loss 2.9264 lr 1.23e-04 gnorm 0.07 54,295 tok/s peakVRAM 13.7GB +[02:18:07] step 4580/4768 loss 2.9300 lr 1.16e-04 gnorm 0.07 54,293 tok/s peakVRAM 13.7GB +[02:21:20] step 4600/4768 loss 2.9284 lr 1.10e-04 gnorm 0.07 54,295 tok/s peakVRAM 13.7GB +[02:24:33] step 4620/4768 loss 2.9155 lr 1.04e-04 gnorm 0.07 54,290 tok/s peakVRAM 13.7GB +[02:27:46] step 4640/4768 loss 2.9244 lr 9.78e-05 gnorm 0.07 54,291 tok/s peakVRAM 13.7GB +[02:31:00] step 4660/4768 loss 2.9173 lr 9.18e-05 gnorm 0.07 54,298 tok/s peakVRAM 13.7GB +[02:34:13] step 4680/4768 loss 2.9151 lr 8.58e-05 gnorm 0.07 54,299 tok/s peakVRAM 13.7GB +[02:37:26] step 4700/4768 loss 2.9109 lr 7.99e-05 gnorm 0.07 54,299 tok/s peakVRAM 13.7GB +[02:40:39] step 4720/4768 loss 2.9122 lr 7.41e-05 gnorm 0.07 54,298 tok/s peakVRAM 13.7GB +[02:43:52] step 4740/4768 loss 2.9100 lr 6.83e-05 gnorm 0.07 54,301 tok/s peakVRAM 13.7GB +[02:47:05] step 4760/4768 loss 2.9194 lr 6.26e-05 gnorm 0.07 54,313 tok/s peakVRAM 13.7GB +[02:48:24] ckpt @ 4768 +[02:48:24] DONE diff --git a/R3/run/val_en.json b/R3/run/val_en.json new file mode 100644 index 0000000000000000000000000000000000000000..d27afcf54b291d22ddc0ef2b7bbdf4ceb410cf76 --- /dev/null +++ b/R3/run/val_en.json @@ -0,0 +1 @@ +{"schema": "val_bpb_v0", "ckpt_sha256": "ea6748f1bcd72a38d3707bc33ef319125fe9c5e9d273e2d424d733dbe32d4564", "step": 4768, "val_sha256": "046bc03b426b75f0e84db7cdf3965a9653d94e722b81b0d03fd0cb92884cf0c5", "tokenizer_sha256": "0640d3bd3674d7a6e59c540d945a2ad275a12e1bcaeb6007c37a90751f9815dd", "script_sha256": "ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1", "trainer_sha256": "db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b", "window": 4096, "n_windows": 547, "tokens": 2240512, "bytes": 9799872, "sum_nat": 6795539.94921875, "bpb": 1.0004101874902782, "ce_token": 3.0330299276320547, "P": 183561103, "dtype": "fp32, tf32 off, bez autocast", "torch": "2.6.0+cu124", "device": "NVIDIA GeForce RTX 4090"} \ No newline at end of file diff --git a/R3/run/val_en.out b/R3/run/val_en.out new file mode 100644 index 0000000000000000000000000000000000000000..501671d4622246294cef64c6897d7e2b343b402a --- /dev/null +++ b/R3/run/val_en.out @@ -0,0 +1 @@ +VAL-BPB-OK step 4768 bpb 1.00041 ce 3.03303 bajty 9799872 P 183561103 -> /home/ubuntu/kolektyw/dowody400m/wyniki/R3/val_en.json diff --git a/R3/run/val_pl.json b/R3/run/val_pl.json new file mode 100644 index 0000000000000000000000000000000000000000..aab36f0ac9515348eb42b6601f53a36e9c142413 --- /dev/null +++ b/R3/run/val_pl.json @@ -0,0 +1 @@ +{"schema": "val_bpb_v0", "ckpt_sha256": "ea6748f1bcd72a38d3707bc33ef319125fe9c5e9d273e2d424d733dbe32d4564", "step": 4768, "val_sha256": "99e66265f47d249c72f5e76838aa23e82959b99089f720bea6e9ee2cb1f49ee1", "tokenizer_sha256": "0640d3bd3674d7a6e59c540d945a2ad275a12e1bcaeb6007c37a90751f9815dd", "script_sha256": "ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1", "trainer_sha256": "db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b", "window": 4096, "n_windows": 406, "tokens": 1662976, "bytes": 7234678, "sum_nat": 5449911.37109375, "bpb": 1.0867878443741827, "ce_token": 3.277203862890234, "P": 183561103, "dtype": "fp32, tf32 off, bez autocast", "torch": "2.6.0+cu124", "device": "NVIDIA GeForce RTX 4090"} \ No newline at end of file diff --git a/R3/run/val_pl.out b/R3/run/val_pl.out new file mode 100644 index 0000000000000000000000000000000000000000..7b5770aa901ea4fad44cbdd0a42f7b6e15851a9f --- /dev/null +++ b/R3/run/val_pl.out @@ -0,0 +1 @@ +VAL-BPB-OK step 4768 bpb 1.08679 ce 3.27720 bajty 7234678 P 183561103 -> /home/ubuntu/kolektyw/dowody400m/wyniki/R3/val_pl.json diff --git a/R3/run/wynik.json b/R3/run/wynik.json new file mode 100644 index 0000000000000000000000000000000000000000..13aa2e57a9ec5bbf86debf514278a08eaf011594 --- /dev/null +++ b/R3/run/wynik.json @@ -0,0 +1 @@ +{"ramie": "R3", "ckpt_sha256": "ea6748f1bcd72a38d3707bc33ef319125fe9c5e9d273e2d424d733dbe32d4564", "bpb_pl": 1.0867878443741827, "bpb_en": 1.0004101874902782, "bpb_wazona_40_60": 1.0349612502438401, "val_pl_sha256": "99e66265f47d249c72f5e76838aa23e82959b99089f720bea6e9ee2cb1f49ee1", "val_en_sha256": "046bc03b426b75f0e84db7cdf3965a9653d94e722b81b0d03fd0cb92884cf0c5", "P": 183561103, "step": 4768} diff --git a/VR0/run/SHA256SUMS_kopiarka b/VR0/run/SHA256SUMS_kopiarka new file mode 100644 index 0000000000000000000000000000000000000000..0203a0ec213c3041edda096b85bc326b5652928a --- /dev/null +++ b/VR0/run/SHA256SUMS_kopiarka @@ -0,0 +1,5 @@ +fd93ad481e1ce29f3bf7d2fe7406b8e2b8821a7a8310bf1795a168236181353f ckpt_500.pt +83c73ec8302a95a1560a6c1777d3a03786c4f0ae69071babd3774c5dd129981b ckpt_1000.pt +c5678f18d42e9e6f62adaaa6b2e309fadf470d2a7dc4e614217c00e155a86840 ckpt_1500.pt +ee399f392ba9fe44419a840a75060eafa26d42ce0227d2538ec47ecf353a0ec0 ckpt_2000.pt +09ffda1f09c8b0e3a86b350bbbe59afb2b352a3246047bd3ba5f3a544a5cd918 ckpt_2500.pt diff --git a/VR0/run/args.txt b/VR0/run/args.txt new file mode 100644 index 0000000000000000000000000000000000000000..b070e9b6a195e72f19cfec366b3b1eb9960bc1a8 --- /dev/null +++ b/VR0/run/args.txt @@ -0,0 +1 @@ +--data-dir /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2 --out-dir /home/ubuntu/kolektyw/dowody400m/wyniki/VR0 --run-id dowody_VR0 --vocab 32768 --dtype uint16 --block 4096 --n-layer 16 --n-embd 896 --n-head 14 --batch 128 --grad-accum 8 --norm rmsnorm --pos rope --rope-theta 500000 --ffn swiglu --ffn-mult 2.667 --qk-norm --attn-bias 0 --optimizer muon --muon-lr 0.02 --lr 6e-4 --min-lr 6e-05 --wd 0.1 --schedule wsd --decay-frac 0.2 --warmup 500 --act-ckpt --loss-chunk 1024 --sampler random --seed 1 --data-seed 20261022 --compile --log-every 20 --eval-every 4769 --eval-iters 1 --steps 4768 --ckpt-every 500 \ No newline at end of file diff --git a/VR0/run/metrics.jsonl b/VR0/run/metrics.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f4f90bd2e2897f76e305a823bc2b78f9475daf86 --- /dev/null +++ b/VR0/run/metrics.jsonl @@ -0,0 +1,143 @@ +{"step": 20, "loss": 10.375734758377074, "lr": 2.3999999999999997e-05, "tok_s": 43561.69633077532} +{"step": 40, "loss": 9.556370401382447, "lr": 4.7999999999999994e-05, "tok_s": 54802.01190866277} +{"step": 60, "loss": 8.694424104690551, "lr": 7.199999999999999e-05, "tok_s": 54810.59212020428} +{"step": 80, "loss": 8.043903589248657, "lr": 9.599999999999999e-05, "tok_s": 54800.26970897335} +{"step": 100, "loss": 7.590182209014893, "lr": 0.00011999999999999999, "tok_s": 54779.21734125376} +{"step": 120, "loss": 7.120254778862, "lr": 0.00014399999999999998, "tok_s": 54752.89193243692} +{"step": 140, "loss": 6.706712174415588, "lr": 0.000168, "tok_s": 54738.036033852244} +{"step": 160, "loss": 6.350001239776612, "lr": 0.00019199999999999998, "tok_s": 54719.08941908544} +{"step": 180, "loss": 6.0842972755432125, "lr": 0.00021599999999999996, "tok_s": 54707.15396784646} +{"step": 200, "loss": 5.863085746765137, "lr": 0.00023999999999999998, "tok_s": 54698.27555838915} +{"step": 220, "loss": 5.675773787498474, "lr": 0.00026399999999999997, "tok_s": 54717.31463776012} +{"step": 240, "loss": 5.500416684150696, "lr": 0.00028799999999999995, "tok_s": 54725.91158743708} +{"step": 260, "loss": 5.331201648712158, "lr": 0.000312, "tok_s": 54722.3224447544} +{"step": 280, "loss": 5.173330140113831, "lr": 0.000336, "tok_s": 54719.78778999873} +{"step": 300, "loss": 5.02945716381073, "lr": 0.00035999999999999997, "tok_s": 54720.856281616565} +{"step": 320, "loss": 4.893988060951233, "lr": 0.00038399999999999996, "tok_s": 54727.508570208585} +{"step": 340, "loss": 4.754287338256836, "lr": 0.000408, "tok_s": 54722.05084391708} +{"step": 360, "loss": 4.627293944358826, "lr": 0.00043199999999999993, "tok_s": 54716.79209654429} +{"step": 380, "loss": 4.506833934783936, "lr": 0.00045599999999999997, "tok_s": 54717.68748886389} +{"step": 400, "loss": 4.405667424201965, "lr": 0.00047999999999999996, "tok_s": 54715.12127059121} +{"step": 420, "loss": 4.323906707763672, "lr": 0.000504, "tok_s": 54712.37553794258} +{"step": 440, "loss": 4.249268102645874, "lr": 0.0005279999999999999, "tok_s": 54707.202215371966} +{"step": 460, "loss": 4.179608964920044, "lr": 0.000552, "tok_s": 54708.802597953494} +{"step": 480, "loss": 4.135195469856262, "lr": 0.0005759999999999999, "tok_s": 54714.530975056325} +{"step": 500, "loss": 4.07597029209137, "lr": 0.0006, "tok_s": 54703.74718638301} +{"step": 520, "loss": 4.0399804472923275, "lr": 0.0006, "tok_s": 54201.71037128161} +{"step": 540, "loss": 3.9887672781944277, "lr": 0.0006, "tok_s": 54666.78511969438} +{"step": 560, "loss": 3.937409543991089, "lr": 0.0006, "tok_s": 54665.677562770834} +{"step": 580, "loss": 3.895251727104187, "lr": 0.0006, "tok_s": 54660.9835144938} +{"step": 600, "loss": 3.8538100957870483, "lr": 0.0006, "tok_s": 54647.22601728159} +{"step": 620, "loss": 3.832408595085144, "lr": 0.0006, "tok_s": 54649.489660106556} +{"step": 640, "loss": 3.8059848070144655, "lr": 0.0006, "tok_s": 54653.118493707785} +{"step": 660, "loss": 3.774251198768616, "lr": 0.0006, "tok_s": 54649.32417206457} +{"step": 680, "loss": 3.75222008228302, "lr": 0.0006, "tok_s": 54641.57933923298} +{"step": 700, "loss": 3.720589852333069, "lr": 0.0006, "tok_s": 54645.67072033027} +{"step": 720, "loss": 3.6912384033203125, "lr": 0.0006, "tok_s": 54647.14990032201} +{"step": 740, "loss": 3.6870697498321534, "lr": 0.0006, "tok_s": 54645.50043474646} +{"step": 760, "loss": 3.6618350386619567, "lr": 0.0006, "tok_s": 54634.07973683338} +{"step": 780, "loss": 3.6360883951187133, "lr": 0.0006, "tok_s": 54633.865952290835} +{"step": 800, "loss": 3.6228135347366335, "lr": 0.0006, "tok_s": 54641.13495449183} +{"step": 820, "loss": 3.6063698410987852, "lr": 0.0006, "tok_s": 54635.02882466679} +{"step": 840, "loss": 3.592339313030243, "lr": 0.0006, "tok_s": 54631.22853162776} +{"step": 860, "loss": 3.5710460782051086, "lr": 0.0006, "tok_s": 54634.984505114546} +{"step": 880, "loss": 3.5654065012931824, "lr": 0.0006, "tok_s": 54630.584874868415} +{"step": 900, "loss": 3.547339415550232, "lr": 0.0006, "tok_s": 54635.75939499837} +{"step": 920, "loss": 3.540403950214386, "lr": 0.0006, "tok_s": 54639.856421294055} +{"step": 940, "loss": 3.5346657276153564, "lr": 0.0006, "tok_s": 54638.11861087071} +{"step": 960, "loss": 3.509232497215271, "lr": 0.0006, "tok_s": 54630.135986696194} +{"step": 980, "loss": 3.5068020462989806, "lr": 0.0006, "tok_s": 54639.62460276463} +{"step": 1000, "loss": 3.4964327454566955, "lr": 0.0006, "tok_s": 54636.49548269226} +{"step": 1020, "loss": 3.490617907047272, "lr": 0.0006, "tok_s": 54035.40671656221} +{"step": 1040, "loss": 3.4636317014694216, "lr": 0.0006, "tok_s": 54679.92097295801} +{"step": 1060, "loss": 3.4751248121261598, "lr": 0.0006, "tok_s": 54675.715978437795} +{"step": 1080, "loss": 3.465067172050476, "lr": 0.0006, "tok_s": 54669.96657174488} +{"step": 1100, "loss": 3.449025547504425, "lr": 0.0006, "tok_s": 54677.20854523376} +{"step": 1120, "loss": 3.4309099674224854, "lr": 0.0006, "tok_s": 54668.019860748704} +{"step": 1140, "loss": 3.4240749716758727, "lr": 0.0006, "tok_s": 54670.18804634113} +{"step": 1160, "loss": 3.4194581270217896, "lr": 0.0006, "tok_s": 54669.003833898685} +{"step": 1180, "loss": 3.421360421180725, "lr": 0.0006, "tok_s": 54674.14274708785} +{"step": 1200, "loss": 3.4032581448554993, "lr": 0.0006, "tok_s": 54675.59091050565} +{"step": 1220, "loss": 3.400775122642517, "lr": 0.0006, "tok_s": 54664.90889028993} +{"step": 1240, "loss": 3.3927265405654907, "lr": 0.0006, "tok_s": 54663.84380537845} +{"step": 1260, "loss": 3.378507101535797, "lr": 0.0006, "tok_s": 54667.782638819524} +{"step": 1280, "loss": 3.3813574433326723, "lr": 0.0006, "tok_s": 54669.03210326399} +{"step": 1300, "loss": 3.3747870206832884, "lr": 0.0006, "tok_s": 54668.31511692797} +{"step": 1320, "loss": 3.3671963930130007, "lr": 0.0006, "tok_s": 54673.84334992449} +{"step": 1340, "loss": 3.367549180984497, "lr": 0.0006, "tok_s": 54665.69026884425} +{"step": 1360, "loss": 3.358504223823547, "lr": 0.0006, "tok_s": 54661.27366741161} +{"step": 1380, "loss": 3.3499217867851256, "lr": 0.0006, "tok_s": 54661.28664318183} +{"step": 1400, "loss": 3.3391000390052796, "lr": 0.0006, "tok_s": 54656.56237545118} +{"step": 1420, "loss": 3.3271182656288145, "lr": 0.0006, "tok_s": 54657.88890019194} +{"step": 1440, "loss": 3.3343973636627195, "lr": 0.0006, "tok_s": 54655.127309719835} +{"step": 1460, "loss": 3.3319170594215395, "lr": 0.0006, "tok_s": 54661.09710243843} +{"step": 1480, "loss": 3.3359232544898987, "lr": 0.0006, "tok_s": 54661.4579103389} +{"step": 1500, "loss": 3.308581829071045, "lr": 0.0006, "tok_s": 54663.928529610734} +{"step": 1520, "loss": 3.316060793399811, "lr": 0.0006, "tok_s": 53964.127696082745} +{"step": 1540, "loss": 3.299985718727112, "lr": 0.0006, "tok_s": 54622.193209558245} +{"step": 1560, "loss": 3.3087693452835083, "lr": 0.0006, "tok_s": 54621.138742895666} +{"step": 1580, "loss": 3.297246742248535, "lr": 0.0006, "tok_s": 54645.40327466757} +{"step": 1600, "loss": 3.2864357590675355, "lr": 0.0006, "tok_s": 54680.79632723692} +{"step": 1620, "loss": 3.2873030424118044, "lr": 0.0006, "tok_s": 54610.05770515974} +{"step": 1640, "loss": 3.2835344433784486, "lr": 0.0006, "tok_s": 54602.530213570935} +{"step": 1660, "loss": 3.2846927642822266, "lr": 0.0006, "tok_s": 54609.03204994534} +{"step": 1680, "loss": 3.2738327860832213, "lr": 0.0006, "tok_s": 54612.34241164312} +{"step": 1700, "loss": 3.2667564272880556, "lr": 0.0006, "tok_s": 54611.02731905383} +{"step": 1720, "loss": 3.2687233567237852, "lr": 0.0006, "tok_s": 54603.805034530065} +{"step": 1740, "loss": 3.262153160572052, "lr": 0.0006, "tok_s": 54616.7359161885} +{"step": 1760, "loss": 3.2620850086212156, "lr": 0.0006, "tok_s": 54604.5965313625} +{"step": 1780, "loss": 3.2486656665802003, "lr": 0.0006, "tok_s": 54604.51504172111} +{"step": 1800, "loss": 3.252673101425171, "lr": 0.0006, "tok_s": 54603.215782580985} +{"step": 1820, "loss": 3.247729980945587, "lr": 0.0006, "tok_s": 54613.012561562886} +{"step": 1840, "loss": 3.241509401798248, "lr": 0.0006, "tok_s": 54609.59755916206} +{"step": 1860, "loss": 3.240514397621155, "lr": 0.0006, "tok_s": 54619.421590640006} +{"step": 1880, "loss": 3.236587393283844, "lr": 0.0006, "tok_s": 54625.87025467581} +{"step": 1900, "loss": 3.231264328956604, "lr": 0.0006, "tok_s": 54622.258199249845} +{"step": 1920, "loss": 3.2316967368125917, "lr": 0.0006, "tok_s": 54615.78346223774} +{"step": 1940, "loss": 3.2239279985427856, "lr": 0.0006, "tok_s": 54614.3597605017} +{"step": 1960, "loss": 3.2283305764198302, "lr": 0.0006, "tok_s": 54609.64821145838} +{"step": 1980, "loss": 3.2229691624641417, "lr": 0.0006, "tok_s": 54609.85583938994} +{"step": 2000, "loss": 3.2164681196212768, "lr": 0.0006, "tok_s": 54609.58718460694} +{"step": 2020, "loss": 3.2158807635307314, "lr": 0.0006, "tok_s": 53972.41612749769} +{"step": 2040, "loss": 3.208861601352692, "lr": 0.0006, "tok_s": 54628.769145588085} +{"step": 2060, "loss": 3.20954909324646, "lr": 0.0006, "tok_s": 54628.65840615726} +{"step": 2080, "loss": 3.197783350944519, "lr": 0.0006, "tok_s": 54623.79466030333} +{"step": 2100, "loss": 3.206951212882996, "lr": 0.0006, "tok_s": 54618.801613549505} +{"step": 2120, "loss": 3.196195650100708, "lr": 0.0006, "tok_s": 54623.245071117766} +{"step": 2140, "loss": 3.2090746760368347, "lr": 0.0006, "tok_s": 54625.91347387303} +{"step": 2160, "loss": 3.1877464652061462, "lr": 0.0006, "tok_s": 54621.16404581991} +{"step": 2180, "loss": 3.1903236269950868, "lr": 0.0006, "tok_s": 54628.99965061038} +{"step": 2200, "loss": 3.1907971262931825, "lr": 0.0006, "tok_s": 54630.87314435991} +{"step": 2220, "loss": 3.1834498405456544, "lr": 0.0006, "tok_s": 54628.1121796269} +{"step": 2240, "loss": 3.180230236053467, "lr": 0.0006, "tok_s": 54630.74882437561} +{"step": 2260, "loss": 3.1768353819847106, "lr": 0.0006, "tok_s": 54631.877496860325} +{"step": 2280, "loss": 3.173125660419464, "lr": 0.0006, "tok_s": 54629.84874285204} +{"step": 2300, "loss": 3.1768848299980164, "lr": 0.0006, "tok_s": 54627.3160655398} +{"step": 2320, "loss": 3.173204851150513, "lr": 0.0006, "tok_s": 54626.434077206} +{"step": 2340, "loss": 3.168735456466675, "lr": 0.0006, "tok_s": 54630.32619494849} +{"step": 2360, "loss": 3.158804166316986, "lr": 0.0006, "tok_s": 54630.44990227507} +{"step": 2380, "loss": 3.158117401599884, "lr": 0.0006, "tok_s": 54632.36075286746} +{"step": 2400, "loss": 3.158198297023773, "lr": 0.0006, "tok_s": 54630.19868809288} +{"step": 2420, "loss": 3.161284792423248, "lr": 0.0006, "tok_s": 54632.13015161673} +{"step": 2440, "loss": 3.15466570854187, "lr": 0.0006, "tok_s": 54634.01044331109} +{"step": 2460, "loss": 3.1544418811798094, "lr": 0.0006, "tok_s": 54628.25046550963} +{"step": 2480, "loss": 3.150985825061798, "lr": 0.0006, "tok_s": 54632.7845674823} +{"step": 2500, "loss": 3.145690131187439, "lr": 0.0006, "tok_s": 54630.282425896905} +{"step": 2520, "loss": 3.147460234165192, "lr": 0.0006, "tok_s": 53970.86694829401} +{"step": 2540, "loss": 3.1477890968322755, "lr": 0.0006, "tok_s": 54625.84474384738} +{"step": 2560, "loss": 3.144875001907349, "lr": 0.0006, "tok_s": 54619.0735470635} +{"step": 2580, "loss": 3.140310990810394, "lr": 0.0006, "tok_s": 54626.89477838313} +{"step": 2600, "loss": 3.1308063864707947, "lr": 0.0006, "tok_s": 54623.575528861686} +{"step": 2620, "loss": 3.1396723985671997, "lr": 0.0006, "tok_s": 54625.477824780835} +{"step": 2640, "loss": 3.140640068054199, "lr": 0.0006, "tok_s": 54624.93613966608} +{"step": 2660, "loss": 3.138874876499176, "lr": 0.0006, "tok_s": 54628.09162000932} +{"step": 2680, "loss": 3.134067153930664, "lr": 0.0006, "tok_s": 54628.929352140134} +{"step": 2700, "loss": 3.13673712015152, "lr": 0.0006, "tok_s": 54629.529405615714} +{"step": 2720, "loss": 3.1201440334320067, "lr": 0.0006, "tok_s": 54629.315317400054} +{"step": 2740, "loss": 3.12159686088562, "lr": 0.0006, "tok_s": 54629.38541329351} +{"step": 2760, "loss": 3.1249221682548525, "lr": 0.0006, "tok_s": 54631.82313872909} +{"step": 2780, "loss": 3.115387523174286, "lr": 0.0006, "tok_s": 54629.63919857502} +{"step": 2800, "loss": 3.1101686716079713, "lr": 0.0006, "tok_s": 54628.71900065539} +{"step": 2820, "loss": 3.1079711079597474, "lr": 0.0006, "tok_s": 54619.04960269675} +{"step": 2840, "loss": 3.116371750831604, "lr": 0.0006, "tok_s": 54611.63159057109} +{"step": 2860, "loss": 3.107532286643982, "lr": 0.0006, "tok_s": 54619.67976030565} diff --git a/VR0/run/srodowisko_VR0_mini0_resume0.txt b/VR0/run/srodowisko_VR0_mini0_resume0.txt new file mode 100644 index 0000000000000000000000000000000000000000..4a155654f7206dd9368caeb9c1a3887bdaf76ac4 --- /dev/null +++ b/VR0/run/srodowisko_VR0_mini0_resume0.txt @@ -0,0 +1,64 @@ +data 2026-10-23T02:33:05Z ARM VR0 GPU 3 MINI 0 DATA_SEED 20261022 CKPT 500 RESUME 0 DEST /var/lib/docker/kolektyw/dowody400m_ckpt/wyniki_VR0 +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +0702c32c0ad602d8d12b16539e05cfbc68d1823cfd39b82b984357d78f27f664 /home/ubuntu/kolektyw/dowody400m/kod/ramie_v03.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +341aae81ce8512294dcd7de93ced26053025677b4ab449606be989f7fd898d27 /home/ubuntu/kolektyw/dowody400m/kod/kopiarka_ckpt.sh +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 diff --git a/VR0/run/srodowisko_resume0.txt b/VR0/run/srodowisko_resume0.txt new file mode 100644 index 0000000000000000000000000000000000000000..4a155654f7206dd9368caeb9c1a3887bdaf76ac4 --- /dev/null +++ b/VR0/run/srodowisko_resume0.txt @@ -0,0 +1,64 @@ +data 2026-10-23T02:33:05Z ARM VR0 GPU 3 MINI 0 DATA_SEED 20261022 CKPT 500 RESUME 0 DEST /var/lib/docker/kolektyw/dowody400m_ckpt/wyniki_VR0 +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +0702c32c0ad602d8d12b16539e05cfbc68d1823cfd39b82b984357d78f27f664 /home/ubuntu/kolektyw/dowody400m/kod/ramie_v03.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +341aae81ce8512294dcd7de93ced26053025677b4ab449606be989f7fd898d27 /home/ubuntu/kolektyw/dowody400m/kod/kopiarka_ckpt.sh +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 diff --git a/VR0/run/train.log b/VR0/run/train.log new file mode 100644 index 0000000000000000000000000000000000000000..40cd868574d2df0b783af9e36dda5bceec11cb1a --- /dev/null +++ b/VR0/run/train.log @@ -0,0 +1,156 @@ +[04:33:11] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[04:33:11] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[04:33:13] model GPT-ref: 183.6M param (L16 d896 h14) +[04:33:14] optimizer=muon muon_lr=0.02 (mult=33.3x) +[04:33:14] r11 pamiec: act_ckpt=True loss_chunk=1024 +[04:33:14] r13: mixer_layout=AAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[04:33:15] torch.compile enabled +[04:33:15] START gpt-ref: steps=4768 (od 0) lr=0.0006->6e-05 +[04:37:15] step 20/4768 loss 10.3757 lr 2.40e-05 gnorm 2.72 43,562 tok/s peakVRAM 13.2GB +[04:40:27] step 40/4768 loss 9.5564 lr 4.80e-05 gnorm 1.29 54,802 tok/s peakVRAM 13.2GB +[04:43:38] step 60/4768 loss 8.6944 lr 7.20e-05 gnorm 0.71 54,811 tok/s peakVRAM 13.2GB +[04:46:49] step 80/4768 loss 8.0439 lr 9.60e-05 gnorm 1.17 54,800 tok/s peakVRAM 13.2GB +[04:50:01] step 100/4768 loss 7.5902 lr 1.20e-04 gnorm 0.90 54,779 tok/s peakVRAM 13.2GB +[04:53:12] step 120/4768 loss 7.1203 lr 1.44e-04 gnorm 1.05 54,753 tok/s peakVRAM 13.2GB +[04:56:24] step 140/4768 loss 6.7067 lr 1.68e-04 gnorm 0.96 54,738 tok/s peakVRAM 13.2GB +[04:59:35] step 160/4768 loss 6.3500 lr 1.92e-04 gnorm 0.76 54,719 tok/s peakVRAM 13.2GB +[05:02:47] step 180/4768 loss 6.0843 lr 2.16e-04 gnorm 0.72 54,707 tok/s peakVRAM 13.2GB +[05:05:59] step 200/4768 loss 5.8631 lr 2.40e-04 gnorm 0.79 54,698 tok/s peakVRAM 13.2GB +[05:09:10] step 220/4768 loss 5.6758 lr 2.64e-04 gnorm 0.73 54,717 tok/s peakVRAM 13.2GB +[05:12:22] step 240/4768 loss 5.5004 lr 2.88e-04 gnorm 0.74 54,726 tok/s peakVRAM 13.2GB +[05:15:34] step 260/4768 loss 5.3312 lr 3.12e-04 gnorm 0.65 54,722 tok/s peakVRAM 13.2GB +[05:18:45] step 280/4768 loss 5.1733 lr 3.36e-04 gnorm 0.82 54,720 tok/s peakVRAM 13.2GB +[05:21:57] step 300/4768 loss 5.0295 lr 3.60e-04 gnorm 0.71 54,721 tok/s peakVRAM 13.2GB +[05:25:09] step 320/4768 loss 4.8940 lr 3.84e-04 gnorm 0.77 54,728 tok/s peakVRAM 13.2GB +[05:28:20] step 340/4768 loss 4.7543 lr 4.08e-04 gnorm 0.60 54,722 tok/s peakVRAM 13.2GB +[05:31:32] step 360/4768 loss 4.6273 lr 4.32e-04 gnorm 0.51 54,717 tok/s peakVRAM 13.2GB +[05:34:43] step 380/4768 loss 4.5068 lr 4.56e-04 gnorm 0.51 54,718 tok/s peakVRAM 13.2GB +[05:37:55] step 400/4768 loss 4.4057 lr 4.80e-04 gnorm 0.47 54,715 tok/s peakVRAM 13.2GB +[05:41:07] step 420/4768 loss 4.3239 lr 5.04e-04 gnorm 0.49 54,712 tok/s peakVRAM 13.2GB +[05:44:18] step 440/4768 loss 4.2493 lr 5.28e-04 gnorm 0.40 54,707 tok/s peakVRAM 13.2GB +[05:47:30] step 460/4768 loss 4.1796 lr 5.52e-04 gnorm 0.39 54,709 tok/s peakVRAM 13.2GB +[05:50:42] step 480/4768 loss 4.1352 lr 5.76e-04 gnorm 0.36 54,715 tok/s peakVRAM 13.2GB +[05:53:53] step 500/4768 loss 4.0760 lr 6.00e-04 gnorm 0.30 54,704 tok/s peakVRAM 13.2GB +[05:53:55] ckpt @ 500 +[05:57:07] step 520/4768 loss 4.0400 lr 6.00e-04 gnorm 0.31 54,202 tok/s peakVRAM 13.2GB +[06:00:19] step 540/4768 loss 3.9888 lr 6.00e-04 gnorm 0.27 54,667 tok/s peakVRAM 13.2GB +[06:03:30] step 560/4768 loss 3.9374 lr 6.00e-04 gnorm 0.21 54,666 tok/s peakVRAM 13.2GB +[06:06:42] step 580/4768 loss 3.8953 lr 6.00e-04 gnorm 0.24 54,661 tok/s peakVRAM 13.2GB +[06:09:54] step 600/4768 loss 3.8538 lr 6.00e-04 gnorm 0.24 54,647 tok/s peakVRAM 13.2GB +[06:13:06] step 620/4768 loss 3.8324 lr 6.00e-04 gnorm 0.20 54,649 tok/s peakVRAM 13.2GB +[06:16:18] step 640/4768 loss 3.8060 lr 6.00e-04 gnorm 0.21 54,653 tok/s peakVRAM 13.2GB +[06:19:30] step 660/4768 loss 3.7743 lr 6.00e-04 gnorm 0.16 54,649 tok/s peakVRAM 13.2GB +[06:22:42] step 680/4768 loss 3.7522 lr 6.00e-04 gnorm 0.18 54,642 tok/s peakVRAM 13.2GB +[06:25:54] step 700/4768 loss 3.7206 lr 6.00e-04 gnorm 0.16 54,646 tok/s peakVRAM 13.2GB +[06:29:05] step 720/4768 loss 3.6912 lr 6.00e-04 gnorm 0.16 54,647 tok/s peakVRAM 13.2GB +[06:32:17] step 740/4768 loss 3.6871 lr 6.00e-04 gnorm 0.17 54,646 tok/s peakVRAM 13.2GB +[06:35:29] step 760/4768 loss 3.6618 lr 6.00e-04 gnorm 0.15 54,634 tok/s peakVRAM 13.2GB +[06:38:41] step 780/4768 loss 3.6361 lr 6.00e-04 gnorm 0.16 54,634 tok/s peakVRAM 13.2GB +[06:41:53] step 800/4768 loss 3.6228 lr 6.00e-04 gnorm 0.14 54,641 tok/s peakVRAM 13.2GB +[06:45:05] step 820/4768 loss 3.6064 lr 6.00e-04 gnorm 0.17 54,635 tok/s peakVRAM 13.2GB +[06:48:17] step 840/4768 loss 3.5923 lr 6.00e-04 gnorm 0.15 54,631 tok/s peakVRAM 13.2GB +[06:51:29] step 860/4768 loss 3.5710 lr 6.00e-04 gnorm 0.13 54,635 tok/s peakVRAM 13.2GB +[06:54:41] step 880/4768 loss 3.5654 lr 6.00e-04 gnorm 0.14 54,631 tok/s peakVRAM 13.2GB +[06:57:53] step 900/4768 loss 3.5473 lr 6.00e-04 gnorm 0.15 54,636 tok/s peakVRAM 13.2GB +[07:01:05] step 920/4768 loss 3.5404 lr 6.00e-04 gnorm 0.14 54,640 tok/s peakVRAM 13.2GB +[07:04:17] step 940/4768 loss 3.5347 lr 6.00e-04 gnorm 0.13 54,638 tok/s peakVRAM 13.2GB +[07:07:29] step 960/4768 loss 3.5092 lr 6.00e-04 gnorm 0.13 54,630 tok/s peakVRAM 13.2GB +[07:10:40] step 980/4768 loss 3.5068 lr 6.00e-04 gnorm 0.14 54,640 tok/s peakVRAM 13.2GB +[07:13:52] step 1000/4768 loss 3.4964 lr 6.00e-04 gnorm 0.12 54,636 tok/s peakVRAM 13.2GB +[07:13:55] ckpt @ 1000 +[07:17:06] step 1020/4768 loss 3.4906 lr 6.00e-04 gnorm 0.14 54,035 tok/s peakVRAM 13.2GB +[07:20:18] step 1040/4768 loss 3.4636 lr 6.00e-04 gnorm 0.13 54,680 tok/s peakVRAM 13.2GB +[07:23:30] step 1060/4768 loss 3.4751 lr 6.00e-04 gnorm 0.12 54,676 tok/s peakVRAM 13.2GB +[07:26:42] step 1080/4768 loss 3.4651 lr 6.00e-04 gnorm 0.14 54,670 tok/s peakVRAM 13.2GB +[07:29:54] step 1100/4768 loss 3.4490 lr 6.00e-04 gnorm 0.13 54,677 tok/s peakVRAM 13.2GB +[07:33:05] step 1120/4768 loss 3.4309 lr 6.00e-04 gnorm 0.12 54,668 tok/s peakVRAM 13.2GB +[07:36:17] step 1140/4768 loss 3.4241 lr 6.00e-04 gnorm 0.13 54,670 tok/s peakVRAM 13.2GB +[07:39:29] step 1160/4768 loss 3.4195 lr 6.00e-04 gnorm 0.14 54,669 tok/s peakVRAM 13.2GB +[07:42:41] step 1180/4768 loss 3.4214 lr 6.00e-04 gnorm 0.13 54,674 tok/s peakVRAM 13.2GB +[07:45:53] step 1200/4768 loss 3.4033 lr 6.00e-04 gnorm 0.11 54,676 tok/s peakVRAM 13.2GB +[07:49:04] step 1220/4768 loss 3.4008 lr 6.00e-04 gnorm 0.11 54,665 tok/s peakVRAM 13.2GB +[07:52:16] step 1240/4768 loss 3.3927 lr 6.00e-04 gnorm 0.14 54,664 tok/s peakVRAM 13.2GB +[07:55:28] step 1260/4768 loss 3.3785 lr 6.00e-04 gnorm 0.13 54,668 tok/s peakVRAM 13.2GB +[07:58:40] step 1280/4768 loss 3.3814 lr 6.00e-04 gnorm 0.11 54,669 tok/s peakVRAM 13.2GB +[08:01:52] step 1300/4768 loss 3.3748 lr 6.00e-04 gnorm 0.11 54,668 tok/s peakVRAM 13.2GB +[08:05:03] step 1320/4768 loss 3.3672 lr 6.00e-04 gnorm 0.12 54,674 tok/s peakVRAM 13.2GB +[08:08:15] step 1340/4768 loss 3.3675 lr 6.00e-04 gnorm 0.13 54,666 tok/s peakVRAM 13.2GB +[08:11:27] step 1360/4768 loss 3.3585 lr 6.00e-04 gnorm 0.13 54,661 tok/s peakVRAM 13.2GB +[08:14:39] step 1380/4768 loss 3.3499 lr 6.00e-04 gnorm 0.11 54,661 tok/s peakVRAM 13.2GB +[08:17:51] step 1400/4768 loss 3.3391 lr 6.00e-04 gnorm 0.11 54,657 tok/s peakVRAM 13.2GB +[08:21:03] step 1420/4768 loss 3.3271 lr 6.00e-04 gnorm 0.13 54,658 tok/s peakVRAM 13.2GB +[08:24:14] step 1440/4768 loss 3.3344 lr 6.00e-04 gnorm 0.13 54,655 tok/s peakVRAM 13.2GB +[08:27:26] step 1460/4768 loss 3.3319 lr 6.00e-04 gnorm 0.11 54,661 tok/s peakVRAM 13.2GB +[08:30:38] step 1480/4768 loss 3.3359 lr 6.00e-04 gnorm 0.12 54,661 tok/s peakVRAM 13.2GB +[08:33:50] step 1500/4768 loss 3.3086 lr 6.00e-04 gnorm 0.12 54,664 tok/s peakVRAM 13.2GB +[08:33:52] ckpt @ 1500 +[08:37:04] step 1520/4768 loss 3.3161 lr 6.00e-04 gnorm 0.12 53,964 tok/s peakVRAM 13.2GB +[08:40:16] step 1540/4768 loss 3.3000 lr 6.00e-04 gnorm 0.11 54,622 tok/s peakVRAM 13.2GB +[08:43:28] step 1560/4768 loss 3.3088 lr 6.00e-04 gnorm 0.11 54,621 tok/s peakVRAM 13.2GB +[08:46:40] step 1580/4768 loss 3.2972 lr 6.00e-04 gnorm 0.11 54,645 tok/s peakVRAM 13.2GB +[08:49:52] step 1600/4768 loss 3.2864 lr 6.00e-04 gnorm 0.12 54,681 tok/s peakVRAM 13.2GB +[08:53:04] step 1620/4768 loss 3.2873 lr 6.00e-04 gnorm 0.13 54,610 tok/s peakVRAM 13.2GB +[08:56:16] step 1640/4768 loss 3.2835 lr 6.00e-04 gnorm 0.11 54,603 tok/s peakVRAM 13.2GB +[08:59:28] step 1660/4768 loss 3.2847 lr 6.00e-04 gnorm 0.11 54,609 tok/s peakVRAM 13.2GB +[09:02:40] step 1680/4768 loss 3.2738 lr 6.00e-04 gnorm 0.10 54,612 tok/s peakVRAM 13.2GB +[09:05:52] step 1700/4768 loss 3.2668 lr 6.00e-04 gnorm 0.11 54,611 tok/s peakVRAM 13.2GB +[09:09:04] step 1720/4768 loss 3.2687 lr 6.00e-04 gnorm 0.11 54,604 tok/s peakVRAM 13.2GB +[09:12:16] step 1740/4768 loss 3.2622 lr 6.00e-04 gnorm 0.11 54,617 tok/s peakVRAM 13.2GB +[09:15:28] step 1760/4768 loss 3.2621 lr 6.00e-04 gnorm 0.12 54,605 tok/s peakVRAM 13.2GB +[09:18:40] step 1780/4768 loss 3.2487 lr 6.00e-04 gnorm 0.11 54,605 tok/s peakVRAM 13.2GB +[09:21:52] step 1800/4768 loss 3.2527 lr 6.00e-04 gnorm 0.11 54,603 tok/s peakVRAM 13.2GB +[09:25:04] step 1820/4768 loss 3.2477 lr 6.00e-04 gnorm 0.11 54,613 tok/s peakVRAM 13.2GB +[09:28:16] step 1840/4768 loss 3.2415 lr 6.00e-04 gnorm 0.10 54,610 tok/s peakVRAM 13.2GB +[09:31:28] step 1860/4768 loss 3.2405 lr 6.00e-04 gnorm 0.11 54,619 tok/s peakVRAM 13.2GB +[09:34:40] step 1880/4768 loss 3.2366 lr 6.00e-04 gnorm 0.10 54,626 tok/s peakVRAM 13.2GB +[09:37:52] step 1900/4768 loss 3.2313 lr 6.00e-04 gnorm 0.11 54,622 tok/s peakVRAM 13.2GB +[09:41:04] step 1920/4768 loss 3.2317 lr 6.00e-04 gnorm 0.11 54,616 tok/s peakVRAM 13.2GB +[09:44:16] step 1940/4768 loss 3.2239 lr 6.00e-04 gnorm 0.11 54,614 tok/s peakVRAM 13.2GB +[09:47:28] step 1960/4768 loss 3.2283 lr 6.00e-04 gnorm 0.10 54,610 tok/s peakVRAM 13.2GB +[09:50:40] step 1980/4768 loss 3.2230 lr 6.00e-04 gnorm 0.11 54,610 tok/s peakVRAM 13.2GB +[09:53:52] step 2000/4768 loss 3.2165 lr 6.00e-04 gnorm 0.12 54,610 tok/s peakVRAM 13.2GB +[09:53:54] ckpt @ 2000 +[09:57:06] step 2020/4768 loss 3.2159 lr 6.00e-04 gnorm 0.11 53,972 tok/s peakVRAM 13.2GB +[10:00:18] step 2040/4768 loss 3.2089 lr 6.00e-04 gnorm 0.10 54,629 tok/s peakVRAM 13.2GB +[10:03:30] step 2060/4768 loss 3.2095 lr 6.00e-04 gnorm 0.11 54,629 tok/s peakVRAM 13.2GB +[10:06:42] step 2080/4768 loss 3.1978 lr 6.00e-04 gnorm 0.10 54,624 tok/s peakVRAM 13.2GB +[10:09:54] step 2100/4768 loss 3.2070 lr 6.00e-04 gnorm 0.10 54,619 tok/s peakVRAM 13.2GB +[10:13:06] step 2120/4768 loss 3.1962 lr 6.00e-04 gnorm 0.11 54,623 tok/s peakVRAM 13.2GB +[10:16:18] step 2140/4768 loss 3.2091 lr 6.00e-04 gnorm 0.10 54,626 tok/s peakVRAM 13.2GB +[10:19:30] step 2160/4768 loss 3.1877 lr 6.00e-04 gnorm 0.11 54,621 tok/s peakVRAM 13.2GB +[10:22:42] step 2180/4768 loss 3.1903 lr 6.00e-04 gnorm 0.10 54,629 tok/s peakVRAM 13.2GB +[10:25:54] step 2200/4768 loss 3.1908 lr 6.00e-04 gnorm 0.11 54,631 tok/s peakVRAM 13.2GB +[10:29:06] step 2220/4768 loss 3.1834 lr 6.00e-04 gnorm 0.10 54,628 tok/s peakVRAM 13.2GB +[10:32:18] step 2240/4768 loss 3.1802 lr 6.00e-04 gnorm 0.10 54,631 tok/s peakVRAM 13.2GB +[10:35:30] step 2260/4768 loss 3.1768 lr 6.00e-04 gnorm 0.11 54,632 tok/s peakVRAM 13.2GB +[10:38:42] step 2280/4768 loss 3.1731 lr 6.00e-04 gnorm 0.10 54,630 tok/s peakVRAM 13.2GB +[10:41:54] step 2300/4768 loss 3.1769 lr 6.00e-04 gnorm 0.10 54,627 tok/s peakVRAM 13.2GB +[10:45:06] step 2320/4768 loss 3.1732 lr 6.00e-04 gnorm 0.11 54,626 tok/s peakVRAM 13.2GB +[10:48:17] step 2340/4768 loss 3.1687 lr 6.00e-04 gnorm 0.10 54,630 tok/s peakVRAM 13.2GB +[10:51:29] step 2360/4768 loss 3.1588 lr 6.00e-04 gnorm 0.10 54,630 tok/s peakVRAM 13.2GB +[10:54:41] step 2380/4768 loss 3.1581 lr 6.00e-04 gnorm 0.10 54,632 tok/s peakVRAM 13.2GB +[10:57:53] step 2400/4768 loss 3.1582 lr 6.00e-04 gnorm 0.10 54,630 tok/s peakVRAM 13.2GB +[11:01:05] step 2420/4768 loss 3.1613 lr 6.00e-04 gnorm 0.09 54,632 tok/s peakVRAM 13.2GB +[11:04:17] step 2440/4768 loss 3.1547 lr 6.00e-04 gnorm 0.11 54,634 tok/s peakVRAM 13.2GB +[11:07:29] step 2460/4768 loss 3.1544 lr 6.00e-04 gnorm 0.10 54,628 tok/s peakVRAM 13.2GB +[11:10:41] step 2480/4768 loss 3.1510 lr 6.00e-04 gnorm 0.11 54,633 tok/s peakVRAM 13.2GB +[11:13:53] step 2500/4768 loss 3.1457 lr 6.00e-04 gnorm 0.10 54,630 tok/s peakVRAM 13.2GB +[11:13:55] ckpt @ 2500 +[11:17:07] step 2520/4768 loss 3.1475 lr 6.00e-04 gnorm 0.10 53,971 tok/s peakVRAM 13.2GB +[11:20:19] step 2540/4768 loss 3.1478 lr 6.00e-04 gnorm 0.10 54,626 tok/s peakVRAM 13.2GB +[11:23:31] step 2560/4768 loss 3.1449 lr 6.00e-04 gnorm 0.11 54,619 tok/s peakVRAM 13.2GB +[11:26:43] step 2580/4768 loss 3.1403 lr 6.00e-04 gnorm 0.10 54,627 tok/s peakVRAM 13.2GB +[11:29:55] step 2600/4768 loss 3.1308 lr 6.00e-04 gnorm 0.10 54,624 tok/s peakVRAM 13.2GB +[11:33:07] step 2620/4768 loss 3.1397 lr 6.00e-04 gnorm 0.10 54,625 tok/s peakVRAM 13.2GB +[11:36:19] step 2640/4768 loss 3.1406 lr 6.00e-04 gnorm 0.10 54,625 tok/s peakVRAM 13.2GB +[11:39:31] step 2660/4768 loss 3.1389 lr 6.00e-04 gnorm 0.09 54,628 tok/s peakVRAM 13.2GB +[11:42:43] step 2680/4768 loss 3.1341 lr 6.00e-04 gnorm 0.10 54,629 tok/s peakVRAM 13.2GB +[11:45:55] step 2700/4768 loss 3.1367 lr 6.00e-04 gnorm 0.10 54,630 tok/s peakVRAM 13.2GB +[11:49:07] step 2720/4768 loss 3.1201 lr 6.00e-04 gnorm 0.11 54,629 tok/s peakVRAM 13.2GB +[11:52:19] step 2740/4768 loss 3.1216 lr 6.00e-04 gnorm 0.10 54,629 tok/s peakVRAM 13.2GB +[11:55:31] step 2760/4768 loss 3.1249 lr 6.00e-04 gnorm 0.10 54,632 tok/s peakVRAM 13.2GB +[11:58:43] step 2780/4768 loss 3.1154 lr 6.00e-04 gnorm 0.10 54,630 tok/s peakVRAM 13.2GB +[12:01:55] step 2800/4768 loss 3.1102 lr 6.00e-04 gnorm 0.10 54,629 tok/s peakVRAM 13.2GB +[12:05:07] step 2820/4768 loss 3.1080 lr 6.00e-04 gnorm 0.09 54,619 tok/s peakVRAM 13.2GB +[12:08:19] step 2840/4768 loss 3.1164 lr 6.00e-04 gnorm 0.10 54,612 tok/s peakVRAM 13.2GB +[12:11:31] step 2860/4768 loss 3.1075 lr 6.00e-04 gnorm 0.10 54,620 tok/s peakVRAM 13.2GB diff --git a/VR0/run/train.out b/VR0/run/train.out new file mode 100644 index 0000000000000000000000000000000000000000..40cd868574d2df0b783af9e36dda5bceec11cb1a --- /dev/null +++ b/VR0/run/train.out @@ -0,0 +1,156 @@ +[04:33:11] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[04:33:11] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[04:33:13] model GPT-ref: 183.6M param (L16 d896 h14) +[04:33:14] optimizer=muon muon_lr=0.02 (mult=33.3x) +[04:33:14] r11 pamiec: act_ckpt=True loss_chunk=1024 +[04:33:14] r13: mixer_layout=AAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[04:33:15] torch.compile enabled +[04:33:15] START gpt-ref: steps=4768 (od 0) lr=0.0006->6e-05 +[04:37:15] step 20/4768 loss 10.3757 lr 2.40e-05 gnorm 2.72 43,562 tok/s peakVRAM 13.2GB +[04:40:27] step 40/4768 loss 9.5564 lr 4.80e-05 gnorm 1.29 54,802 tok/s peakVRAM 13.2GB +[04:43:38] step 60/4768 loss 8.6944 lr 7.20e-05 gnorm 0.71 54,811 tok/s peakVRAM 13.2GB +[04:46:49] step 80/4768 loss 8.0439 lr 9.60e-05 gnorm 1.17 54,800 tok/s peakVRAM 13.2GB +[04:50:01] step 100/4768 loss 7.5902 lr 1.20e-04 gnorm 0.90 54,779 tok/s peakVRAM 13.2GB +[04:53:12] step 120/4768 loss 7.1203 lr 1.44e-04 gnorm 1.05 54,753 tok/s peakVRAM 13.2GB +[04:56:24] step 140/4768 loss 6.7067 lr 1.68e-04 gnorm 0.96 54,738 tok/s peakVRAM 13.2GB +[04:59:35] step 160/4768 loss 6.3500 lr 1.92e-04 gnorm 0.76 54,719 tok/s peakVRAM 13.2GB +[05:02:47] step 180/4768 loss 6.0843 lr 2.16e-04 gnorm 0.72 54,707 tok/s peakVRAM 13.2GB +[05:05:59] step 200/4768 loss 5.8631 lr 2.40e-04 gnorm 0.79 54,698 tok/s peakVRAM 13.2GB +[05:09:10] step 220/4768 loss 5.6758 lr 2.64e-04 gnorm 0.73 54,717 tok/s peakVRAM 13.2GB +[05:12:22] step 240/4768 loss 5.5004 lr 2.88e-04 gnorm 0.74 54,726 tok/s peakVRAM 13.2GB +[05:15:34] step 260/4768 loss 5.3312 lr 3.12e-04 gnorm 0.65 54,722 tok/s peakVRAM 13.2GB +[05:18:45] step 280/4768 loss 5.1733 lr 3.36e-04 gnorm 0.82 54,720 tok/s peakVRAM 13.2GB +[05:21:57] step 300/4768 loss 5.0295 lr 3.60e-04 gnorm 0.71 54,721 tok/s peakVRAM 13.2GB +[05:25:09] step 320/4768 loss 4.8940 lr 3.84e-04 gnorm 0.77 54,728 tok/s peakVRAM 13.2GB +[05:28:20] step 340/4768 loss 4.7543 lr 4.08e-04 gnorm 0.60 54,722 tok/s peakVRAM 13.2GB +[05:31:32] step 360/4768 loss 4.6273 lr 4.32e-04 gnorm 0.51 54,717 tok/s peakVRAM 13.2GB +[05:34:43] step 380/4768 loss 4.5068 lr 4.56e-04 gnorm 0.51 54,718 tok/s peakVRAM 13.2GB +[05:37:55] step 400/4768 loss 4.4057 lr 4.80e-04 gnorm 0.47 54,715 tok/s peakVRAM 13.2GB +[05:41:07] step 420/4768 loss 4.3239 lr 5.04e-04 gnorm 0.49 54,712 tok/s peakVRAM 13.2GB +[05:44:18] step 440/4768 loss 4.2493 lr 5.28e-04 gnorm 0.40 54,707 tok/s peakVRAM 13.2GB +[05:47:30] step 460/4768 loss 4.1796 lr 5.52e-04 gnorm 0.39 54,709 tok/s peakVRAM 13.2GB +[05:50:42] step 480/4768 loss 4.1352 lr 5.76e-04 gnorm 0.36 54,715 tok/s peakVRAM 13.2GB +[05:53:53] step 500/4768 loss 4.0760 lr 6.00e-04 gnorm 0.30 54,704 tok/s peakVRAM 13.2GB +[05:53:55] ckpt @ 500 +[05:57:07] step 520/4768 loss 4.0400 lr 6.00e-04 gnorm 0.31 54,202 tok/s peakVRAM 13.2GB +[06:00:19] step 540/4768 loss 3.9888 lr 6.00e-04 gnorm 0.27 54,667 tok/s peakVRAM 13.2GB +[06:03:30] step 560/4768 loss 3.9374 lr 6.00e-04 gnorm 0.21 54,666 tok/s peakVRAM 13.2GB +[06:06:42] step 580/4768 loss 3.8953 lr 6.00e-04 gnorm 0.24 54,661 tok/s peakVRAM 13.2GB +[06:09:54] step 600/4768 loss 3.8538 lr 6.00e-04 gnorm 0.24 54,647 tok/s peakVRAM 13.2GB +[06:13:06] step 620/4768 loss 3.8324 lr 6.00e-04 gnorm 0.20 54,649 tok/s peakVRAM 13.2GB +[06:16:18] step 640/4768 loss 3.8060 lr 6.00e-04 gnorm 0.21 54,653 tok/s peakVRAM 13.2GB +[06:19:30] step 660/4768 loss 3.7743 lr 6.00e-04 gnorm 0.16 54,649 tok/s peakVRAM 13.2GB +[06:22:42] step 680/4768 loss 3.7522 lr 6.00e-04 gnorm 0.18 54,642 tok/s peakVRAM 13.2GB +[06:25:54] step 700/4768 loss 3.7206 lr 6.00e-04 gnorm 0.16 54,646 tok/s peakVRAM 13.2GB +[06:29:05] step 720/4768 loss 3.6912 lr 6.00e-04 gnorm 0.16 54,647 tok/s peakVRAM 13.2GB +[06:32:17] step 740/4768 loss 3.6871 lr 6.00e-04 gnorm 0.17 54,646 tok/s peakVRAM 13.2GB +[06:35:29] step 760/4768 loss 3.6618 lr 6.00e-04 gnorm 0.15 54,634 tok/s peakVRAM 13.2GB +[06:38:41] step 780/4768 loss 3.6361 lr 6.00e-04 gnorm 0.16 54,634 tok/s peakVRAM 13.2GB +[06:41:53] step 800/4768 loss 3.6228 lr 6.00e-04 gnorm 0.14 54,641 tok/s peakVRAM 13.2GB +[06:45:05] step 820/4768 loss 3.6064 lr 6.00e-04 gnorm 0.17 54,635 tok/s peakVRAM 13.2GB +[06:48:17] step 840/4768 loss 3.5923 lr 6.00e-04 gnorm 0.15 54,631 tok/s peakVRAM 13.2GB +[06:51:29] step 860/4768 loss 3.5710 lr 6.00e-04 gnorm 0.13 54,635 tok/s peakVRAM 13.2GB +[06:54:41] step 880/4768 loss 3.5654 lr 6.00e-04 gnorm 0.14 54,631 tok/s peakVRAM 13.2GB +[06:57:53] step 900/4768 loss 3.5473 lr 6.00e-04 gnorm 0.15 54,636 tok/s peakVRAM 13.2GB +[07:01:05] step 920/4768 loss 3.5404 lr 6.00e-04 gnorm 0.14 54,640 tok/s peakVRAM 13.2GB +[07:04:17] step 940/4768 loss 3.5347 lr 6.00e-04 gnorm 0.13 54,638 tok/s peakVRAM 13.2GB +[07:07:29] step 960/4768 loss 3.5092 lr 6.00e-04 gnorm 0.13 54,630 tok/s peakVRAM 13.2GB +[07:10:40] step 980/4768 loss 3.5068 lr 6.00e-04 gnorm 0.14 54,640 tok/s peakVRAM 13.2GB +[07:13:52] step 1000/4768 loss 3.4964 lr 6.00e-04 gnorm 0.12 54,636 tok/s peakVRAM 13.2GB +[07:13:55] ckpt @ 1000 +[07:17:06] step 1020/4768 loss 3.4906 lr 6.00e-04 gnorm 0.14 54,035 tok/s peakVRAM 13.2GB +[07:20:18] step 1040/4768 loss 3.4636 lr 6.00e-04 gnorm 0.13 54,680 tok/s peakVRAM 13.2GB +[07:23:30] step 1060/4768 loss 3.4751 lr 6.00e-04 gnorm 0.12 54,676 tok/s peakVRAM 13.2GB +[07:26:42] step 1080/4768 loss 3.4651 lr 6.00e-04 gnorm 0.14 54,670 tok/s peakVRAM 13.2GB +[07:29:54] step 1100/4768 loss 3.4490 lr 6.00e-04 gnorm 0.13 54,677 tok/s peakVRAM 13.2GB +[07:33:05] step 1120/4768 loss 3.4309 lr 6.00e-04 gnorm 0.12 54,668 tok/s peakVRAM 13.2GB +[07:36:17] step 1140/4768 loss 3.4241 lr 6.00e-04 gnorm 0.13 54,670 tok/s peakVRAM 13.2GB +[07:39:29] step 1160/4768 loss 3.4195 lr 6.00e-04 gnorm 0.14 54,669 tok/s peakVRAM 13.2GB +[07:42:41] step 1180/4768 loss 3.4214 lr 6.00e-04 gnorm 0.13 54,674 tok/s peakVRAM 13.2GB +[07:45:53] step 1200/4768 loss 3.4033 lr 6.00e-04 gnorm 0.11 54,676 tok/s peakVRAM 13.2GB +[07:49:04] step 1220/4768 loss 3.4008 lr 6.00e-04 gnorm 0.11 54,665 tok/s peakVRAM 13.2GB +[07:52:16] step 1240/4768 loss 3.3927 lr 6.00e-04 gnorm 0.14 54,664 tok/s peakVRAM 13.2GB +[07:55:28] step 1260/4768 loss 3.3785 lr 6.00e-04 gnorm 0.13 54,668 tok/s peakVRAM 13.2GB +[07:58:40] step 1280/4768 loss 3.3814 lr 6.00e-04 gnorm 0.11 54,669 tok/s peakVRAM 13.2GB +[08:01:52] step 1300/4768 loss 3.3748 lr 6.00e-04 gnorm 0.11 54,668 tok/s peakVRAM 13.2GB +[08:05:03] step 1320/4768 loss 3.3672 lr 6.00e-04 gnorm 0.12 54,674 tok/s peakVRAM 13.2GB +[08:08:15] step 1340/4768 loss 3.3675 lr 6.00e-04 gnorm 0.13 54,666 tok/s peakVRAM 13.2GB +[08:11:27] step 1360/4768 loss 3.3585 lr 6.00e-04 gnorm 0.13 54,661 tok/s peakVRAM 13.2GB +[08:14:39] step 1380/4768 loss 3.3499 lr 6.00e-04 gnorm 0.11 54,661 tok/s peakVRAM 13.2GB +[08:17:51] step 1400/4768 loss 3.3391 lr 6.00e-04 gnorm 0.11 54,657 tok/s peakVRAM 13.2GB +[08:21:03] step 1420/4768 loss 3.3271 lr 6.00e-04 gnorm 0.13 54,658 tok/s peakVRAM 13.2GB +[08:24:14] step 1440/4768 loss 3.3344 lr 6.00e-04 gnorm 0.13 54,655 tok/s peakVRAM 13.2GB +[08:27:26] step 1460/4768 loss 3.3319 lr 6.00e-04 gnorm 0.11 54,661 tok/s peakVRAM 13.2GB +[08:30:38] step 1480/4768 loss 3.3359 lr 6.00e-04 gnorm 0.12 54,661 tok/s peakVRAM 13.2GB +[08:33:50] step 1500/4768 loss 3.3086 lr 6.00e-04 gnorm 0.12 54,664 tok/s peakVRAM 13.2GB +[08:33:52] ckpt @ 1500 +[08:37:04] step 1520/4768 loss 3.3161 lr 6.00e-04 gnorm 0.12 53,964 tok/s peakVRAM 13.2GB +[08:40:16] step 1540/4768 loss 3.3000 lr 6.00e-04 gnorm 0.11 54,622 tok/s peakVRAM 13.2GB +[08:43:28] step 1560/4768 loss 3.3088 lr 6.00e-04 gnorm 0.11 54,621 tok/s peakVRAM 13.2GB +[08:46:40] step 1580/4768 loss 3.2972 lr 6.00e-04 gnorm 0.11 54,645 tok/s peakVRAM 13.2GB +[08:49:52] step 1600/4768 loss 3.2864 lr 6.00e-04 gnorm 0.12 54,681 tok/s peakVRAM 13.2GB +[08:53:04] step 1620/4768 loss 3.2873 lr 6.00e-04 gnorm 0.13 54,610 tok/s peakVRAM 13.2GB +[08:56:16] step 1640/4768 loss 3.2835 lr 6.00e-04 gnorm 0.11 54,603 tok/s peakVRAM 13.2GB +[08:59:28] step 1660/4768 loss 3.2847 lr 6.00e-04 gnorm 0.11 54,609 tok/s peakVRAM 13.2GB +[09:02:40] step 1680/4768 loss 3.2738 lr 6.00e-04 gnorm 0.10 54,612 tok/s peakVRAM 13.2GB +[09:05:52] step 1700/4768 loss 3.2668 lr 6.00e-04 gnorm 0.11 54,611 tok/s peakVRAM 13.2GB +[09:09:04] step 1720/4768 loss 3.2687 lr 6.00e-04 gnorm 0.11 54,604 tok/s peakVRAM 13.2GB +[09:12:16] step 1740/4768 loss 3.2622 lr 6.00e-04 gnorm 0.11 54,617 tok/s peakVRAM 13.2GB +[09:15:28] step 1760/4768 loss 3.2621 lr 6.00e-04 gnorm 0.12 54,605 tok/s peakVRAM 13.2GB +[09:18:40] step 1780/4768 loss 3.2487 lr 6.00e-04 gnorm 0.11 54,605 tok/s peakVRAM 13.2GB +[09:21:52] step 1800/4768 loss 3.2527 lr 6.00e-04 gnorm 0.11 54,603 tok/s peakVRAM 13.2GB +[09:25:04] step 1820/4768 loss 3.2477 lr 6.00e-04 gnorm 0.11 54,613 tok/s peakVRAM 13.2GB +[09:28:16] step 1840/4768 loss 3.2415 lr 6.00e-04 gnorm 0.10 54,610 tok/s peakVRAM 13.2GB +[09:31:28] step 1860/4768 loss 3.2405 lr 6.00e-04 gnorm 0.11 54,619 tok/s peakVRAM 13.2GB +[09:34:40] step 1880/4768 loss 3.2366 lr 6.00e-04 gnorm 0.10 54,626 tok/s peakVRAM 13.2GB +[09:37:52] step 1900/4768 loss 3.2313 lr 6.00e-04 gnorm 0.11 54,622 tok/s peakVRAM 13.2GB +[09:41:04] step 1920/4768 loss 3.2317 lr 6.00e-04 gnorm 0.11 54,616 tok/s peakVRAM 13.2GB +[09:44:16] step 1940/4768 loss 3.2239 lr 6.00e-04 gnorm 0.11 54,614 tok/s peakVRAM 13.2GB +[09:47:28] step 1960/4768 loss 3.2283 lr 6.00e-04 gnorm 0.10 54,610 tok/s peakVRAM 13.2GB +[09:50:40] step 1980/4768 loss 3.2230 lr 6.00e-04 gnorm 0.11 54,610 tok/s peakVRAM 13.2GB +[09:53:52] step 2000/4768 loss 3.2165 lr 6.00e-04 gnorm 0.12 54,610 tok/s peakVRAM 13.2GB +[09:53:54] ckpt @ 2000 +[09:57:06] step 2020/4768 loss 3.2159 lr 6.00e-04 gnorm 0.11 53,972 tok/s peakVRAM 13.2GB +[10:00:18] step 2040/4768 loss 3.2089 lr 6.00e-04 gnorm 0.10 54,629 tok/s peakVRAM 13.2GB +[10:03:30] step 2060/4768 loss 3.2095 lr 6.00e-04 gnorm 0.11 54,629 tok/s peakVRAM 13.2GB +[10:06:42] step 2080/4768 loss 3.1978 lr 6.00e-04 gnorm 0.10 54,624 tok/s peakVRAM 13.2GB +[10:09:54] step 2100/4768 loss 3.2070 lr 6.00e-04 gnorm 0.10 54,619 tok/s peakVRAM 13.2GB +[10:13:06] step 2120/4768 loss 3.1962 lr 6.00e-04 gnorm 0.11 54,623 tok/s peakVRAM 13.2GB +[10:16:18] step 2140/4768 loss 3.2091 lr 6.00e-04 gnorm 0.10 54,626 tok/s peakVRAM 13.2GB +[10:19:30] step 2160/4768 loss 3.1877 lr 6.00e-04 gnorm 0.11 54,621 tok/s peakVRAM 13.2GB +[10:22:42] step 2180/4768 loss 3.1903 lr 6.00e-04 gnorm 0.10 54,629 tok/s peakVRAM 13.2GB +[10:25:54] step 2200/4768 loss 3.1908 lr 6.00e-04 gnorm 0.11 54,631 tok/s peakVRAM 13.2GB +[10:29:06] step 2220/4768 loss 3.1834 lr 6.00e-04 gnorm 0.10 54,628 tok/s peakVRAM 13.2GB +[10:32:18] step 2240/4768 loss 3.1802 lr 6.00e-04 gnorm 0.10 54,631 tok/s peakVRAM 13.2GB +[10:35:30] step 2260/4768 loss 3.1768 lr 6.00e-04 gnorm 0.11 54,632 tok/s peakVRAM 13.2GB +[10:38:42] step 2280/4768 loss 3.1731 lr 6.00e-04 gnorm 0.10 54,630 tok/s peakVRAM 13.2GB +[10:41:54] step 2300/4768 loss 3.1769 lr 6.00e-04 gnorm 0.10 54,627 tok/s peakVRAM 13.2GB +[10:45:06] step 2320/4768 loss 3.1732 lr 6.00e-04 gnorm 0.11 54,626 tok/s peakVRAM 13.2GB +[10:48:17] step 2340/4768 loss 3.1687 lr 6.00e-04 gnorm 0.10 54,630 tok/s peakVRAM 13.2GB +[10:51:29] step 2360/4768 loss 3.1588 lr 6.00e-04 gnorm 0.10 54,630 tok/s peakVRAM 13.2GB +[10:54:41] step 2380/4768 loss 3.1581 lr 6.00e-04 gnorm 0.10 54,632 tok/s peakVRAM 13.2GB +[10:57:53] step 2400/4768 loss 3.1582 lr 6.00e-04 gnorm 0.10 54,630 tok/s peakVRAM 13.2GB +[11:01:05] step 2420/4768 loss 3.1613 lr 6.00e-04 gnorm 0.09 54,632 tok/s peakVRAM 13.2GB +[11:04:17] step 2440/4768 loss 3.1547 lr 6.00e-04 gnorm 0.11 54,634 tok/s peakVRAM 13.2GB +[11:07:29] step 2460/4768 loss 3.1544 lr 6.00e-04 gnorm 0.10 54,628 tok/s peakVRAM 13.2GB +[11:10:41] step 2480/4768 loss 3.1510 lr 6.00e-04 gnorm 0.11 54,633 tok/s peakVRAM 13.2GB +[11:13:53] step 2500/4768 loss 3.1457 lr 6.00e-04 gnorm 0.10 54,630 tok/s peakVRAM 13.2GB +[11:13:55] ckpt @ 2500 +[11:17:07] step 2520/4768 loss 3.1475 lr 6.00e-04 gnorm 0.10 53,971 tok/s peakVRAM 13.2GB +[11:20:19] step 2540/4768 loss 3.1478 lr 6.00e-04 gnorm 0.10 54,626 tok/s peakVRAM 13.2GB +[11:23:31] step 2560/4768 loss 3.1449 lr 6.00e-04 gnorm 0.11 54,619 tok/s peakVRAM 13.2GB +[11:26:43] step 2580/4768 loss 3.1403 lr 6.00e-04 gnorm 0.10 54,627 tok/s peakVRAM 13.2GB +[11:29:55] step 2600/4768 loss 3.1308 lr 6.00e-04 gnorm 0.10 54,624 tok/s peakVRAM 13.2GB +[11:33:07] step 2620/4768 loss 3.1397 lr 6.00e-04 gnorm 0.10 54,625 tok/s peakVRAM 13.2GB +[11:36:19] step 2640/4768 loss 3.1406 lr 6.00e-04 gnorm 0.10 54,625 tok/s peakVRAM 13.2GB +[11:39:31] step 2660/4768 loss 3.1389 lr 6.00e-04 gnorm 0.09 54,628 tok/s peakVRAM 13.2GB +[11:42:43] step 2680/4768 loss 3.1341 lr 6.00e-04 gnorm 0.10 54,629 tok/s peakVRAM 13.2GB +[11:45:55] step 2700/4768 loss 3.1367 lr 6.00e-04 gnorm 0.10 54,630 tok/s peakVRAM 13.2GB +[11:49:07] step 2720/4768 loss 3.1201 lr 6.00e-04 gnorm 0.11 54,629 tok/s peakVRAM 13.2GB +[11:52:19] step 2740/4768 loss 3.1216 lr 6.00e-04 gnorm 0.10 54,629 tok/s peakVRAM 13.2GB +[11:55:31] step 2760/4768 loss 3.1249 lr 6.00e-04 gnorm 0.10 54,632 tok/s peakVRAM 13.2GB +[11:58:43] step 2780/4768 loss 3.1154 lr 6.00e-04 gnorm 0.10 54,630 tok/s peakVRAM 13.2GB +[12:01:55] step 2800/4768 loss 3.1102 lr 6.00e-04 gnorm 0.10 54,629 tok/s peakVRAM 13.2GB +[12:05:07] step 2820/4768 loss 3.1080 lr 6.00e-04 gnorm 0.09 54,619 tok/s peakVRAM 13.2GB +[12:08:19] step 2840/4768 loss 3.1164 lr 6.00e-04 gnorm 0.10 54,612 tok/s peakVRAM 13.2GB +[12:11:31] step 2860/4768 loss 3.1075 lr 6.00e-04 gnorm 0.10 54,620 tok/s peakVRAM 13.2GB diff --git a/kod/kolejka_v03.sh b/kod/kolejka_v03.sh new file mode 100644 index 0000000000000000000000000000000000000000..f8da29147400b43353dd0237b92ade6195c553d7 --- /dev/null +++ b/kod/kolejka_v03.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# kolejka_v03.sh (Monter, 22.10): następca kolejka.sh (1d889c19) dla ramie_v03 (ckpt co 500 + historia + RESUME). +# 1) pierwsza wolna karta po fali 1 -> test_wznowienia.sh (tmux test_resume); 2) ramiona startują dopiero, gdy test dał +# TEST-WZNOWIENIA-OK i jest plik OK_V03 (przegląd Harta albo decyzja operatora); ROZJAZD = STOP kolejki. +# Karta wolna / padnięcie / plik STOP_V03 / KOLEJKA-KONIEC: jak w kolejka.sh. Po AW2 stop (zasada Harta 12:34Z). +# użycie (tmux kolejka_v03): RAMIE_V03_SHA=<8> bash kolejka_v03.sh +set -uo pipefail +: "${RAMIE_V03_SHA:?}" +B="$HOME/kolektyw/dowody400m" +KOLEJKA=(K12 GQA VR0 MU2 AW1 AW2) +declare -A NA_KARCIE=([0]=fala1_R1 [1]=fala1_R2 [2]=fala1_R3 [3]=fala1_K22) +exec >> "$B/kolejka_v03.log" 2>&1 +echo "KOLEJKA-V03-START $(date -u +%FT%TZ) ${KOLEJKA[*]} ramie_v03 $RAMIE_V03_SHA" +wolna() { + local g=$1 s=${NA_KARCIE[$1]} + if [ "$s" = test_resume ]; then tmux has-session -t test_resume 2>/dev/null && return 1 + else grep -q RAMIE-OK "$B/$s.log" 2>/dev/null || return 1; fi + [ -z "$(nvidia-smi -i "$g" --query-compute-apps=pid --format=csv,noheader)" ] || return 1 + [ "$(nvidia-smi -i "$g" --query-gpu=memory.used --format=csv,noheader,nounits)" -lt 500 ] || return 1 +} +padlo() { + local s=${NA_KARCIE[$1]} + [ "$s" = test_resume ] && return 1 + ! grep -q RAMIE-OK "$B/$s.log" 2>/dev/null && ! tmux has-session -t "$s" 2>/dev/null +} +TEST=nie # nie -> trwa -> ok +i=0 +while [ $i -lt ${#KOLEJKA[@]} ]; do + [ -e "$B/STOP_V03" ] && { echo "STOP-PLIK $(date -u +%FT%TZ): przed ${KOLEJKA[$i]}"; exit 0; } + for g in 0 1 2 3; do + if padlo "$g"; then echo "STOP $(date -u +%FT%TZ): ${NA_KARCIE[$g]} na GPU $g bez RAMIE-OK"; exit 1; fi + done + if [ "$TEST" = trwa ] && ! tmux has-session -t test_resume 2>/dev/null; then + if grep -q TEST-WZNOWIENIA-OK "$B/test_resume/test.log" 2>/dev/null; then TEST=ok; echo "TEST-OK $(date -u +%FT%TZ)" + else echo "STOP $(date -u +%FT%TZ): test wznowienia bez OK (log $B/test_resume/test.log)"; exit 1; fi + fi + for g in 0 1 2 3; do + wolna "$g" || continue + if [ "$TEST" = nie ]; then + tmux new-session -d -s test_resume "RAMIE_V03_SHA=$RAMIE_V03_SHA bash $B/kod/test_wznowienia.sh $g" + NA_KARCIE[$g]=test_resume; TEST=trwa; echo "TEST-START GPU $g $(date -u +%FT%TZ)"; sleep 30; continue + fi + [ "$TEST" = ok ] && [ -e "$B/OK_V03" ] || continue + [ $i -lt ${#KOLEJKA[@]} ] || break + a=${KOLEJKA[$i]} + tmux new-session -d -s "kolejka_$a" "MB=16 RAMIE_V03_SHA=$RAMIE_V03_SHA bash $B/kod/uruchom_ramie_v03.sh $a $g 0 > $B/kolejka_$a.log 2>&1" + NA_KARCIE[$g]=kolejka_$a + echo "START $a GPU $g $(date -u +%FT%TZ) ramie_v03" + i=$((i + 1)); sleep 30 + done + sleep 60 +done +echo "KOLEJKA-KONIEC $(date -u +%FT%TZ): wszystkie ramiona fazy 1 uruchomione" diff --git a/kod/kopiarka_ckpt.sh b/kod/kopiarka_ckpt.sh new file mode 100644 index 0000000000000000000000000000000000000000..d24645fe65937e86ab5355b0e8b4f673fff56e47 --- /dev/null +++ b/kod/kopiarka_ckpt.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# kopiarka_ckpt.sh (Monter, 22.10): historia checkpointów ramienia jak w GoLLeM v4, bez zmian w trenerze. +# r23 nadpisuje jeden OUT/ckpt.pt; po każdej linii „ckpt @ N” w OUT/train.log kopia trafia do DEST/ckpt_N.pt +# (.part -> sha256 -> mv; wpis w DEST/SHA256SUMS). Kopiowany jest tylko NAJNOWSZY krok z logu: starszy krok bez kopii +# (ckpt.pt już nadpisany) dostaje wpis BRAK w DEST/kopiarka.log, nigdy kopię z cudzego kroku. +# Koniec: plik OUT/.kopiarka_stop (ustawia uruchom_ramie.sh po ramie.sh) -> ostatnie przejście i wyjście. +# użycie: bash kopiarka_ckpt.sh +set -uo pipefail +OUT=$1; DEST=$2 +mkdir -p "$DEST" || { echo "STOP: mkdir $DEST"; exit 1; } +L="$DEST/kopiarka.log" +przejscie() { + [ -f "$OUT/train.log" ] || return 0 + local kroki ostatni n h + kroki=$(grep -o "ckpt @ [0-9]*" "$OUT/train.log" | awk '{print $3}' | sort -n -u) + [ -n "$kroki" ] || return 0 + ostatni=$(echo "$kroki" | tail -1) + for n in $kroki; do + [ -e "$DEST/ckpt_$n.pt" ] && continue + grep -q "^BRAK ckpt_$n " "$L" 2>/dev/null && continue + if [ "$n" != "$ostatni" ]; then echo "BRAK ckpt_$n $(date -u +%FT%TZ): ckpt.pt nadpisany przed kopią" >> "$L"; continue; fi + cp "$OUT/ckpt.pt" "$DEST/ckpt_$n.pt.part" || { echo "BLAD cp ckpt_$n $(date -u +%FT%TZ)" >> "$L"; return 0; } + # Hart 22:46Z: jeśli w czasie kopiowania r23 zapisał nowszy ckpt, kopia mogłaby mieć cudzy krok -> skasuj, BRAK + if [ "$(grep -o "ckpt @ [0-9]*" "$OUT/train.log" | awk '{print $3}' | sort -n | tail -1)" != "$n" ]; then + rm -f "$DEST/ckpt_$n.pt.part"; echo "BRAK ckpt_$n $(date -u +%FT%TZ): nowszy ckpt w trakcie kopiowania" >> "$L"; continue + fi + h=$(sha256sum "$DEST/ckpt_$n.pt.part" | cut -d' ' -f1) + mv "$DEST/ckpt_$n.pt.part" "$DEST/ckpt_$n.pt" + echo "$h ckpt_$n.pt" >> "$DEST/SHA256SUMS" + echo "KOPIA ckpt_$n.pt $h $(date -u +%FT%TZ)" >> "$L" + done +} +echo "START $(date -u +%FT%TZ) OUT=$OUT" >> "$L" +while [ ! -e "$OUT/.kopiarka_stop" ]; do przejscie; sleep 30; done +przejscie +rm -f "$OUT/.kopiarka_stop" +echo "KONIEC $(date -u +%FT%TZ)" >> "$L" diff --git a/kod/ramie.sh b/kod/ramie.sh new file mode 100644 index 0000000000000000000000000000000000000000..11675807679d817ca21b7f0338da398f49c280ee --- /dev/null +++ b/kod/ramie.sh @@ -0,0 +1,109 @@ +#!/bin/bash +# ramie.sh v0.1 (Hart, 20.10): jedno ramię dnia dowodów 400M na jednej karcie (karta Mierniczego +# Mierniczy/miary/karta_dzien_dowodow_400m_v0.md v0.1; ramiona i P: forum Hart 19:32Z / 19:25Z, decyzja Arka 20:47Z pkt 8). +# Wzorzec R = przepis linii 500M (start_500m.sh: RMSNorm, QK-norm, RoPE θ 500k, value residual, SwiGLU 2,667, bez biasów, +# Muon + AdamW, WSD) przeskalowany do ~180M: 16 × 896 × 14, MHA. Każde ramię zmienia JEDEN czynnik: +# R1 R2 R3 wzorzec, --seed 1 / 2 / 3 (szum ziarna; kolejność danych ta sama: --data-seed stałe) +# K12 12 × 1024 × 16 (płycej, szerzej) K22 22 × 768 × 12 (głębiej, węziej) +# GQA 18 × 896 × 14, KV 2 (oszczędność parametrów na K / V wydana na 2 warstwy) +# VR0 bez value residual +# MU2 Muon, muon-lr × 2 (0,04), AdamW-aux bez zmian (Mierniczy 21:04Z) AW1 / AW2 AdamW 6e-4 / 1,2e-3 +# C1 C2 złożone zwycięzcy (definiuje Mierniczy po fali 1): C_SHAPE z flagami kształtu / architektury, tylko z listy +# --n-layer N --n-embd N --n-head N --kv-heads N --value-residual --no-value-residual; inaczej STOP +# Wspólne: okno 4096, partia 128 × 4096 = 524 288 tok./krok (mikrobatch MB × akumulacja), TOKENS / partia kroków, +# WSD: warmup 500, stały lr, wygaszanie ostatnie 20 % do lr / 10; bf16, compile, act-ckpt, loss-chunk 1024, --sampler +# random, --data-seed DATA_SEED (stałe dla wszystkich ramion). Ocena: val_bpb.py na val_pl.bin i val_en.bin po ostatnim +# kroku (wynik główny karty: bpb ważone 40 : 60 liczy Mierniczy z obu JSON). +# v0.1 (Wartownik 21:12Z): liczby ze środowiska tylko ^[1-9][0-9]*$ (bez wyrażeń bash), argumenty trenera jako tablica, +# C_ARGS zastąpione listą dozwolonych flag kształtu; val.bin w D to plik pętli trenera (pętla eval wyłączona, więc treść +# nie wpływa na wynik; przygotuj_dowody zapisuje go jako mieszankę 40 : 60) - już nie musi być kopią val_en.bin; +# tokenizer osobno (TOK, przypięty pełnym sha tokenizera v6 32k 0640d3bd). +# env: PY, T (r23), V (val_bpb.py), D (train.bin, val.bin, val_pl.bin, val_en.bin z przygotuj_dowody.py), TOK +# (tokenizer.json v6 32k), O (katalog wyników), ARM, GPU, TRAIN_SHA, VALPL_SHA, VALEN_SHA, DATA_SEED (stałe), +# TOKENS (2500000000), MB (8), MINI=1 = 20 kroków na prawdziwym kształcie + val (suchy bieg ścieżki), +# DRY=1 = tylko kontrole wejść i wydruk argumentów trenera (bez GPU, bez zapisu). +set -uo pipefail +: "${PY:?}"; : "${T:?}"; : "${V:?}"; : "${D:?}"; : "${TOK:?}"; : "${O:?}"; : "${ARM:?}"; : "${GPU:?}" +: "${TRAIN_SHA:?}"; : "${VALPL_SHA:?}"; : "${VALEN_SHA:?}"; : "${DATA_SEED:?}" +TOKENS=${TOKENS:-2500000000}; MB=${MB:-8}; MINI=${MINI:-0}; DRY=${DRY:-0} +TOK_SHA=0640d3bd3674d7a6e59c540d945a2ad275a12e1bcaeb6007c37a90751f9815dd +die() { echo "STOP: $*"; exit 1; } +shf() { sha256sum "$1" | cut -d' ' -f1; } +liczba() { [[ $2 =~ ^[1-9][0-9]*$ ]] || die "$1 = '$2' nie jest dodatnią liczbą całkowitą"; } +liczba MB "$MB"; liczba TOKENS "$TOKENS"; liczba DATA_SEED "$DATA_SEED" +[[ $GPU =~ ^[0-9]$ ]] || die "GPU = jedna cyfra" +[[ $MINI =~ ^[01]$ && $DRY =~ ^[01]$ ]] || die "MINI / DRY = 0 albo 1" +[ "$(shf "$T")" = db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b ] || die "T != r23 db36c150" +[ "$(shf "$V")" = ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 ] || die "V != val_bpb.py ce84287c" +for f in train.bin val.bin val_pl.bin val_en.bin; do [ -s "$D/$f" ] || die "brak $D/$f"; done +[ -s "$TOK" ] || die "brak $TOK" +[ "$(shf "$D/train.bin")" = "$TRAIN_SHA" ] || die "train.bin != TRAIN_SHA" +[ "$(shf "$D/val_pl.bin")" = "$VALPL_SHA" ] || die "val_pl.bin != VALPL_SHA" +[ "$(shf "$D/val_en.bin")" = "$VALEN_SHA" ] || die "val_en.bin != VALEN_SHA" +[ "$(shf "$TOK")" = "$TOK_SHA" ] || die "tokenizer != v6 32k 0640d3bd" +[ $(( 128 % MB )) = 0 ] || die "128 % MB != 0" +GA=$(( 128 / MB )); STEPS=$(( TOKENS / (128 * 4096) )) +[ "$MINI" = 1 ] && STEPS=20 +[ "$STEPS" -ge 1 ] || die "TOKENS < jednej partii" +KSZT=(--n-layer 16 --n-embd 896 --n-head 14); ARCH=(--value-residual); OPT=(--optimizer muon --muon-lr 0.02 --lr 6e-4) +LR=6e-4; SEED=1 +case $ARM in + R1) ;; R2) SEED=2 ;; R3) SEED=3 ;; + K12) KSZT=(--n-layer 12 --n-embd 1024 --n-head 16) ;; + K22) KSZT=(--n-layer 22 --n-embd 768 --n-head 12) ;; + GQA) KSZT=(--n-layer 18 --n-embd 896 --n-head 14 --kv-heads 2) ;; + VR0) ARCH=() ;; + MU2) OPT=(--optimizer muon --muon-lr 0.04 --lr 6e-4) ;; + AW1) OPT=(--optimizer adamw --lr 6e-4) ;; + AW2) OPT=(--optimizer adamw --lr 1.2e-3); LR=1.2e-3 ;; + C1|C2) + : "${C_SHAPE:?$ARM: ramię złożone wymaga C_SHAPE z karty (Mierniczy po fali 1)}" + read -r -a CT <<< "$C_SHAPE"; KSZT=(); ARCH=(); i=0 + while [ $i -lt ${#CT[@]} ]; do + case ${CT[$i]} in + --n-layer|--n-embd|--n-head|--kv-heads) + [[ ${CT[$((i + 1))]:-} =~ ^[1-9][0-9]*$ ]] || die "C_SHAPE: ${CT[$i]} bez liczby" + KSZT+=("${CT[$i]}" "${CT[$((i + 1))]}"); i=$(( i + 2 )) ;; + --value-residual) ARCH=(--value-residual); i=$(( i + 1 )) ;; + --no-value-residual) ARCH=(); i=$(( i + 1 )) ;; + *) die "C_SHAPE: niedozwolony token '${CT[$i]}'" ;; + esac + done + [ "$ARM" = C2 ] && SEED=2 ;; + *) die "nieznane ramię $ARM" ;; +esac +MINLR=$(awk "BEGIN{printf \"%.6g\", $LR/10}") +OUT="$O/$ARM"; [ "$MINI" = 1 ] && OUT="$O/mini_$ARM" +ARGS=(--data-dir "$D" --out-dir "$OUT" --run-id "dowody_$ARM" --vocab 32768 --dtype uint16 --block 4096 + "${KSZT[@]}" "${ARCH[@]}" --batch 128 --grad-accum "$GA" --norm rmsnorm --pos rope --rope-theta 500000 + --ffn swiglu --ffn-mult 2.667 --qk-norm --attn-bias 0 "${OPT[@]}" --min-lr "$MINLR" --wd 0.1 --schedule wsd + --decay-frac 0.2 --warmup 500 --act-ckpt --loss-chunk 1024 --sampler random --seed "$SEED" --data-seed "$DATA_SEED" + --compile --log-every 20 --eval-every $(( STEPS + 1 )) --eval-iters 1 --steps "$STEPS" --ckpt-every "$STEPS") +if [ "$DRY" = 1 ]; then + echo "DRY $ARM steps $STEPS ga $GA seed $SEED lr $LR min_lr $MINLR: ${KSZT[*]} ${ARCH[*]} ${OPT[*]}"; exit 0 +fi +[ -e "$OUT/ckpt.pt" ] && die "$OUT/ckpt.pt istnieje (wynik ramienia nie jest nadpisywany)" +mkdir -p "$OUT" || die "mkdir $OUT" +exec 9>"$OUT/.lock"; flock -n 9 || die "$OUT zajęty" +unset HF_TOKEN HUGGING_FACE_HUB_TOKEN; export HF_HUB_OFFLINE=1 OMP_NUM_THREADS=8 CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=$GPU +printf '%q ' "${ARGS[@]}" > "$OUT/args.txt" +echo "RAMIE $ARM $(date -u +%FT%TZ) gpu $GPU steps $STEPS mb $MB ga $GA seed $SEED data_seed $DATA_SEED lr $LR min_lr $MINLR mini $MINI" +"$PY" -u "$T" "${ARGS[@]}" < /dev/null > "$OUT/train.out" 2>&1 || die "trening rc $? (log $OUT/train.out)" +grep -q "ckpt @ $STEPS" "$OUT/train.log" || die "brak ckpt @ $STEPS" +CK="$OUT/ckpt.pt"; CKS=$(shf "$CK") +for L in pl en; do + VS=$VALEN_SHA; [ "$L" = pl ] && VS=$VALPL_SHA + "$PY" "$V" --ckpt "$CK" --ckpt-sha "$CKS" --trainer "$T" --val "$D/val_$L.bin" --val-sha "$VS" --tokenizer "$TOK" \ + --tokenizer-sha "$TOK_SHA" --dtype uint16 --out "$OUT/val_$L.json" < /dev/null > "$OUT/val_$L.out" 2>&1 \ + || die "val_bpb $L rc $? (log $OUT/val_$L.out)" +done +# wynik ramienia (Mierniczy 21:12Z): PL, EN, ważona 40 : 60 i sumy plików val; ramię bez tych liczb = niepoliczone +"$PY" - "$OUT" "$ARM" "$VALPL_SHA" "$VALEN_SHA" "$CKS" > "$OUT/wynik.json" <<'EOF' || die "wynik.json" +import json, sys +out, arm, spl, sen, cks = sys.argv[1:6] +pl, en = (json.load(open(f"{out}/val_{x}.json", encoding="utf-8")) for x in ("pl", "en")) +print(json.dumps({"ramie": arm, "ckpt_sha256": cks, "bpb_pl": pl["bpb"], "bpb_en": en["bpb"], + "bpb_wazona_40_60": 0.4 * pl["bpb"] + 0.6 * en["bpb"], "val_pl_sha256": spl, "val_en_sha256": sen, + "P": pl["P"], "step": pl["step"]})) +EOF +echo "RAMIE-OK $ARM $(date -u +%FT%TZ) ckpt ${CKS:0:8} $(cat "$OUT/wynik.json")" diff --git a/kod/ramie_v03.sh b/kod/ramie_v03.sh new file mode 100644 index 0000000000000000000000000000000000000000..8d426d3683594899c0c399fd94afb78bbe93b7f0 --- /dev/null +++ b/kod/ramie_v03.sh @@ -0,0 +1,122 @@ +#!/bin/bash +# ramie.sh v0.1 (Hart, 20.10): jedno ramię dnia dowodów 400M na jednej karcie (karta Mierniczego +# Mierniczy/miary/karta_dzien_dowodow_400m_v0.md v0.1; ramiona i P: forum Hart 19:32Z / 19:25Z, decyzja Arka 20:47Z pkt 8). +# Wzorzec R = przepis linii 500M (start_500m.sh: RMSNorm, QK-norm, RoPE θ 500k, value residual, SwiGLU 2,667, bez biasów, +# Muon + AdamW, WSD) przeskalowany do ~180M: 16 × 896 × 14, MHA. Każde ramię zmienia JEDEN czynnik: +# R1 R2 R3 wzorzec, --seed 1 / 2 / 3 (szum ziarna; kolejność danych ta sama: --data-seed stałe) +# K12 12 × 1024 × 16 (płycej, szerzej) K22 22 × 768 × 12 (głębiej, węziej) +# GQA 18 × 896 × 14, KV 2 (oszczędność parametrów na K / V wydana na 2 warstwy) +# VR0 bez value residual +# MU2 Muon, muon-lr × 2 (0,04), AdamW-aux bez zmian (Mierniczy 21:04Z) AW1 / AW2 AdamW 6e-4 / 1,2e-3 +# C1 C2 złożone zwycięzcy (definiuje Mierniczy po fali 1): C_SHAPE z flagami kształtu / architektury, tylko z listy +# --n-layer N --n-embd N --n-head N --kv-heads N --value-residual --no-value-residual; inaczej STOP +# Wspólne: okno 4096, partia 128 × 4096 = 524 288 tok./krok (mikrobatch MB × akumulacja), TOKENS / partia kroków, +# WSD: warmup 500, stały lr, wygaszanie ostatnie 20 % do lr / 10; bf16, compile, act-ckpt, loss-chunk 1024, --sampler +# random, --data-seed DATA_SEED (stałe dla wszystkich ramion). Ocena: val_bpb.py na val_pl.bin i val_en.bin po ostatnim +# kroku (wynik główny karty: bpb ważone 40 : 60 liczy Mierniczy z obu JSON). +# v0.1 (Wartownik 21:12Z): liczby ze środowiska tylko ^[1-9][0-9]*$ (bez wyrażeń bash), argumenty trenera jako tablica, +# C_ARGS zastąpione listą dozwolonych flag kształtu; val.bin w D to plik pętli trenera (pętla eval wyłączona, więc treść +# nie wpływa na wynik; przygotuj_dowody zapisuje go jako mieszankę 40 : 60) - już nie musi być kopią val_en.bin; +# tokenizer osobno (TOK, przypięty pełnym sha tokenizera v6 32k 0640d3bd). +# env: PY, T (r23), V (val_bpb.py), D (train.bin, val.bin, val_pl.bin, val_en.bin z przygotuj_dowody.py), TOK +# (tokenizer.json v6 32k), O (katalog wyników), ARM, GPU, TRAIN_SHA, VALPL_SHA, VALEN_SHA, DATA_SEED (stałe), +# TOKENS (2500000000), MB (8), MINI=1 = 20 kroków na prawdziwym kształcie + val (suchy bieg ścieżki), +# DRY=1 = tylko kontrole wejść i wydruk argumentów trenera (bez GPU, bez zapisu). +# v0.3 (Monter, 22.10; Arek: checkpointy do wracania jak w GoLLeM v4; v0.1 + tylko I/O, przepis bez zmian): +# CKPT (500) = --ckpt-every (r23 zapisuje też ckpt po ostatnim kroku poza rytmem), RESUME=1 = --resume z OUT/ckpt.pt +# (r23: STOP przy innym data_seed; RNG samplera random przewijany o krok z ckpt), MINI_STEPS (20) = kroki MINI; +# train.out dopisywany (>>), żeby wznowienie nie kasowało logu przerwanej części. +set -uo pipefail +: "${PY:?}"; : "${T:?}"; : "${V:?}"; : "${D:?}"; : "${TOK:?}"; : "${O:?}"; : "${ARM:?}"; : "${GPU:?}" +: "${TRAIN_SHA:?}"; : "${VALPL_SHA:?}"; : "${VALEN_SHA:?}"; : "${DATA_SEED:?}" +TOKENS=${TOKENS:-2500000000}; MB=${MB:-8}; MINI=${MINI:-0}; DRY=${DRY:-0} +CKPT=${CKPT:-500}; RESUME=${RESUME:-0}; MINI_STEPS=${MINI_STEPS:-20} +TOK_SHA=0640d3bd3674d7a6e59c540d945a2ad275a12e1bcaeb6007c37a90751f9815dd +die() { echo "STOP: $*"; exit 1; } +shf() { sha256sum "$1" | cut -d' ' -f1; } +liczba() { [[ $2 =~ ^[1-9][0-9]*$ ]] || die "$1 = '$2' nie jest dodatnią liczbą całkowitą"; } +liczba MB "$MB"; liczba TOKENS "$TOKENS"; liczba DATA_SEED "$DATA_SEED"; liczba CKPT "$CKPT"; liczba MINI_STEPS "$MINI_STEPS" +[[ $GPU =~ ^[0-9]$ ]] || die "GPU = jedna cyfra" +[[ $MINI =~ ^[01]$ && $DRY =~ ^[01]$ && $RESUME =~ ^[01]$ ]] || die "MINI / DRY / RESUME = 0 albo 1" +[ "$(shf "$T")" = db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b ] || die "T != r23 db36c150" +[ "$(shf "$V")" = ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 ] || die "V != val_bpb.py ce84287c" +for f in train.bin val.bin val_pl.bin val_en.bin; do [ -s "$D/$f" ] || die "brak $D/$f"; done +[ -s "$TOK" ] || die "brak $TOK" +[ "$(shf "$D/train.bin")" = "$TRAIN_SHA" ] || die "train.bin != TRAIN_SHA" +[ "$(shf "$D/val_pl.bin")" = "$VALPL_SHA" ] || die "val_pl.bin != VALPL_SHA" +[ "$(shf "$D/val_en.bin")" = "$VALEN_SHA" ] || die "val_en.bin != VALEN_SHA" +[ "$(shf "$TOK")" = "$TOK_SHA" ] || die "tokenizer != v6 32k 0640d3bd" +[ $(( 128 % MB )) = 0 ] || die "128 % MB != 0" +GA=$(( 128 / MB )); STEPS=$(( TOKENS / (128 * 4096) )) +[ "$MINI" = 1 ] && STEPS=$MINI_STEPS +CK_EVERY=$(( CKPT < STEPS ? CKPT : STEPS )) +[ "$STEPS" -ge 1 ] || die "TOKENS < jednej partii" +KSZT=(--n-layer 16 --n-embd 896 --n-head 14); ARCH=(--value-residual); OPT=(--optimizer muon --muon-lr 0.02 --lr 6e-4) +LR=6e-4; SEED=1 +case $ARM in + R1) ;; R2) SEED=2 ;; R3) SEED=3 ;; + K12) KSZT=(--n-layer 12 --n-embd 1024 --n-head 16) ;; + K22) KSZT=(--n-layer 22 --n-embd 768 --n-head 12) ;; + GQA) KSZT=(--n-layer 18 --n-embd 896 --n-head 14 --kv-heads 2) ;; + VR0) ARCH=() ;; + MU2) OPT=(--optimizer muon --muon-lr 0.04 --lr 6e-4) ;; + AW1) OPT=(--optimizer adamw --lr 6e-4) ;; + AW2) OPT=(--optimizer adamw --lr 1.2e-3); LR=1.2e-3 ;; + C1|C2) + : "${C_SHAPE:?$ARM: ramię złożone wymaga C_SHAPE z karty (Mierniczy po fali 1)}" + read -r -a CT <<< "$C_SHAPE"; KSZT=(); ARCH=(); i=0 + while [ $i -lt ${#CT[@]} ]; do + case ${CT[$i]} in + --n-layer|--n-embd|--n-head|--kv-heads) + [[ ${CT[$((i + 1))]:-} =~ ^[1-9][0-9]*$ ]] || die "C_SHAPE: ${CT[$i]} bez liczby" + KSZT+=("${CT[$i]}" "${CT[$((i + 1))]}"); i=$(( i + 2 )) ;; + --value-residual) ARCH=(--value-residual); i=$(( i + 1 )) ;; + --no-value-residual) ARCH=(); i=$(( i + 1 )) ;; + *) die "C_SHAPE: niedozwolony token '${CT[$i]}'" ;; + esac + done + [ "$ARM" = C2 ] && SEED=2 ;; + *) die "nieznane ramię $ARM" ;; +esac +MINLR=$(awk "BEGIN{printf \"%.6g\", $LR/10}") +OUT="$O/$ARM"; [ "$MINI" = 1 ] && OUT="$O/mini_$ARM" +ARGS=(--data-dir "$D" --out-dir "$OUT" --run-id "dowody_$ARM" --vocab 32768 --dtype uint16 --block 4096 + "${KSZT[@]}" "${ARCH[@]}" --batch 128 --grad-accum "$GA" --norm rmsnorm --pos rope --rope-theta 500000 + --ffn swiglu --ffn-mult 2.667 --qk-norm --attn-bias 0 "${OPT[@]}" --min-lr "$MINLR" --wd 0.1 --schedule wsd + --decay-frac 0.2 --warmup 500 --act-ckpt --loss-chunk 1024 --sampler random --seed "$SEED" --data-seed "$DATA_SEED" + --compile --log-every 20 --eval-every $(( STEPS + 1 )) --eval-iters 1 --steps "$STEPS" --ckpt-every "$CK_EVERY") +if [ "$DRY" = 1 ]; then + echo "DRY $ARM steps $STEPS ga $GA seed $SEED lr $LR min_lr $MINLR ckpt_every $CK_EVERY resume $RESUME: ${KSZT[*]} ${ARCH[*]} ${OPT[*]}"; exit 0 +fi +if [ "$RESUME" = 1 ]; then + # Hart 22:38Z: wznowienie tylko z istniejącego ckpt.pt i z tymi samymi argumentami co zapisany args.txt + [ -s "$OUT/ckpt.pt" ] || die "RESUME=1: brak $OUT/ckpt.pt" + [ "$(printf '%q ' "${ARGS[@]}")" = "$(cat "$OUT/args.txt" 2>/dev/null)" ] || die "RESUME=1: argumenty != $OUT/args.txt" +else + [ -e "$OUT/ckpt.pt" ] && die "$OUT/ckpt.pt istnieje (wynik ramienia nie jest nadpisywany)" +fi +mkdir -p "$OUT" || die "mkdir $OUT" +exec 9>"$OUT/.lock"; flock -n 9 || die "$OUT zajęty" +unset HF_TOKEN HUGGING_FACE_HUB_TOKEN; export HF_HUB_OFFLINE=1 OMP_NUM_THREADS=8 CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=$GPU +[ "$RESUME" = 1 ] || printf '%q ' "${ARGS[@]}" > "$OUT/args.txt" +RES=(); [ "$RESUME" = 1 ] && RES=(--resume) +echo "RAMIE $ARM $(date -u +%FT%TZ) gpu $GPU steps $STEPS mb $MB ga $GA seed $SEED data_seed $DATA_SEED lr $LR min_lr $MINLR mini $MINI ckpt_every $CK_EVERY resume $RESUME" +"$PY" -u "$T" "${ARGS[@]}" "${RES[@]}" < /dev/null >> "$OUT/train.out" 2>&1 || die "trening rc $? (log $OUT/train.out)" +grep -q "ckpt @ $STEPS" "$OUT/train.log" || die "brak ckpt @ $STEPS" +CK="$OUT/ckpt.pt"; CKS=$(shf "$CK") +for L in pl en; do + VS=$VALEN_SHA; [ "$L" = pl ] && VS=$VALPL_SHA + "$PY" "$V" --ckpt "$CK" --ckpt-sha "$CKS" --trainer "$T" --val "$D/val_$L.bin" --val-sha "$VS" --tokenizer "$TOK" \ + --tokenizer-sha "$TOK_SHA" --dtype uint16 --out "$OUT/val_$L.json" < /dev/null > "$OUT/val_$L.out" 2>&1 \ + || die "val_bpb $L rc $? (log $OUT/val_$L.out)" +done +# wynik ramienia (Mierniczy 21:12Z): PL, EN, ważona 40 : 60 i sumy plików val; ramię bez tych liczb = niepoliczone +"$PY" - "$OUT" "$ARM" "$VALPL_SHA" "$VALEN_SHA" "$CKS" > "$OUT/wynik.json" <<'EOF' || die "wynik.json" +import json, sys +out, arm, spl, sen, cks = sys.argv[1:6] +pl, en = (json.load(open(f"{out}/val_{x}.json", encoding="utf-8")) for x in ("pl", "en")) +print(json.dumps({"ramie": arm, "ckpt_sha256": cks, "bpb_pl": pl["bpb"], "bpb_en": en["bpb"], + "bpb_wazona_40_60": 0.4 * pl["bpb"] + 0.6 * en["bpb"], "val_pl_sha256": spl, "val_en_sha256": sen, + "P": pl["P"], "step": pl["step"]})) +EOF +echo "RAMIE-OK $ARM $(date -u +%FT%TZ) ckpt ${CKS:0:8} $(cat "$OUT/wynik.json")" diff --git a/kod/test_wznowienia.sh b/kod/test_wznowienia.sh new file mode 100644 index 0000000000000000000000000000000000000000..1836ce6ca9fb8d5cc8fd117a52a3d4653c99b78a --- /dev/null +++ b/kod/test_wznowienia.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# test_wznowienia.sh (Monter, 22.10; warunki Harta 22:38Z): ramie_v03 R1 MINI 40 kroków ciągiem (A) vs ten sam bieg +# wznowiony z ckpt kroku 20 biegu A (B: 20 -> 40). Twardy warunek: sha tokenów batchy po wznowieniu (kroki 20, 21, 39) +# identyczne z biegiem ciągłym (r23 --batch-sha, bez treningu). Opisowo: strata kroku 40 i bpb A vs B (compile = szum). +# użycie: RAMIE_V03_SHA=<8> bash test_wznowienia.sh +set -euo pipefail +G=$1 +B0="$HOME/kolektyw/dowody400m"; TD="$B0/test_resume"; K="$B0/kod" +DA=/var/lib/docker/kolektyw/dowody400m_ckpt/A_mini_R1 +[ -e "$TD" ] && { echo "STOP: $TD istnieje"; exit 1; } +[ -e "$DA" ] && { echo "STOP: $DA istnieje (stare kopie ckpt z poprzedniego testu)"; exit 1; } +mkdir -p "$TD" +exec > >(tee -a "$TD/test.log") 2>&1 +echo "TEST-START $(date -u +%FT%TZ) GPU $G" +# A: 40 kroków ciągiem, ckpt co 20 (kopiarka trzyma ckpt_20 i ckpt_40) +O="$TD/A" CKPT=20 MINI_STEPS=40 MB=16 bash "$K/uruchom_ramie_v03.sh" R1 "$G" 1 +[ -s "$DA/ckpt_20.pt" ] || { echo "STOP: brak $DA/ckpt_20.pt"; exit 1; } +# B: ten sam bieg w osobnym katalogu, start z ckpt_20 biegu A, RESUME=1 +mkdir -p "$TD/B/mini_R1" +cp "$DA/ckpt_20.pt" "$TD/B/mini_R1/ckpt.pt" +sed "s#$TD/A/mini_R1#$TD/B/mini_R1#g" "$TD/A/mini_R1/args.txt" > "$TD/B/mini_R1/args.txt" +O="$TD/B" CKPT=20 MINI_STEPS=40 MB=16 RESUME=1 bash "$K/uruchom_ramie_v03.sh" R1 "$G" 1 +grep -q "RESUME @ 20" "$TD/B/mini_R1/train.log" || { echo "STOP: B nie wznowił od kroku 20"; exit 1; } +# sha batchy: C = od zera (bez resume), D = z ckpt_20 + --resume; ten sam Prefetcher co trening +R=$HOME/kolektyw/polocr/pydev/root/usr/include; export CPATH=$R/python3.12:$R/x86_64-linux-gnu/python3.12:$R +export CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=$G HF_HUB_OFFLINE=1 OMP_NUM_THREADS=8 +PY="$HOME/kolektyw/polocr/venv/bin/python" +for X in C D; do + mkdir -p "$TD/$X" + eval "ARGS=($(sed "s#$TD/A/mini_R1#$TD/$X#g" "$TD/A/mini_R1/args.txt"))" + RES=(); if [ "$X" = D ]; then cp "$DA/ckpt_20.pt" "$TD/D/ckpt.pt"; RES=(--resume); fi + "$PY" -u "$K/train_gpt_ref_r23.py" "${ARGS[@]}" "${RES[@]}" --batch-sha 20,21,39 < /dev/null > "$TD/$X/batch_sha.out" 2>&1 + grep "BATCH-SHA krok" "$TD/$X/batch_sha.out" | sed 's/^\[[^]]*\] //' > "$TD/$X/batch_sha.txt" +done +echo "== sha batchy (C ciągły | D wznowiony)"; paste "$TD/C/batch_sha.txt" "$TD/D/batch_sha.txt" +echo "== straty"; grep -E "step +(20|40)/40" "$TD/A/mini_R1/train.log" "$TD/B/mini_R1/train.log" +echo "== wynik A"; cat "$TD/A/mini_R1/wynik.json"; echo; echo "== wynik B"; cat "$TD/B/mini_R1/wynik.json"; echo +if [ "$(wc -l < "$TD/C/batch_sha.txt")" = 3 ] && cmp -s "$TD/C/batch_sha.txt" "$TD/D/batch_sha.txt"; then + echo "TEST-WZNOWIENIA-OK $(date -u +%FT%TZ)" +else + echo "TEST-WZNOWIENIA-ROZJAZD $(date -u +%FT%TZ)"; exit 3 +fi diff --git a/kod/train_gpt_ref_r23.py b/kod/train_gpt_ref_r23.py new file mode 100644 index 0000000000000000000000000000000000000000..d127e6c035b67112fb02e0d2e38b285d92bb4b53 --- /dev/null +++ b/kod/train_gpt_ref_r23.py @@ -0,0 +1,1321 @@ +#!/usr/bin/env python +# NOTE (GoLLeM-v5 leaderboard repo): this is a GENERAL nanoGPT-style causal transformer; +# vocab/dtype are CLI-parameterized. The crown/Path-B leaderboard checkpoints were trained in +# BPE-12k mode: --vocab 12288 --dtype uint16 (NOT the byte-level default below). The header +# doc-comment reflects the file origin as a standard-GPT control vs experimental BDH; the +# leaderboard models are the STANDARD transformer in BPE mode and do NOT use BDH. +# -*- coding: utf-8 -*- +""" +Referencyjny ZWYKLY transformer (byte-level nanoGPT-style) ~25M — apples-to-apples vs BDH-25M. +Ta sama data (train.bin/val.bin uint8), ten sam scale (~25M), ten sam byte-level (vocab256). +Rozni sie TYLKO architektura (standard causal transformer vs BDH fast-weights) -> czysta referencja. + +CLI mirror train_bdh.py. GPU ROCm/CUDA bf16, cosine+warmup+clip, ckpt/resume, logging. +Autor: Hart (N-02). + +Smoke throughput (bez danych PII, syntetyczny bufor): --synthetic --steps 60 +Realny: --data-dir . --run-id gpt25m_run1 --steps 30000 +r8 (Monter): wiele GPU przez torchrun (DDP). --batch = batch GLOBALNY; kazdy rank bierze swoje wiersze z tego samego +batcha (te same starty okien co przebieg 1-GPU), gradienty usredniane przez DDP, wiec krok = krok 1-GPU z tym samym +--batch (do numeryki). Logi, metryki, zdarzenia i ckpt pisze tylko rank 0. Bez torchrun (WORLD_SIZE=1) = r7 bez zmian. + torchrun --standalone --nproc-per-node 4 train_gpt_ref_r8.py ... --batch 64 +r11 (Monter, 10.10, Wzrost etap 1b: dlugie okna): dwie opcje tylko na pamiec, domyslnie WYLACZONE (bez nich = r10 bit w bit): + --act-ckpt przeliczanie aktywacji po bloku w treningu (torch.utils.checkpoint, use_reentrant=False); + --loss-chunk N strata liczona kawalkami po N pozycji (glowica + CE w checkpoint), bez pelnych logitow [B,T,V]; + forward zwraca wtedy logits=None, a strata = ta sama srednia po B*T (i ten sam z-loss). + Okno i baza RoPE z CLI dzialaja z --init-from bez zmian (RoPE bez parametrow, --pos rope nie ma pos-embeddingu). +r12 (Monter, 10.10, Slayer9-Eff): --loop K = wspoldzielenie wag miedzy warstwami (looped transformer, pomysl Kacpra 18:30Z): + n_layer unikalnych blokow przechodzi sie K razy (glebokosc liczenia n_layer*K, parametry n_layer). Tablica liczy parametry + modelu, wiec K nie zmienia rozmiaru. Blok 0 w kazdym przejsciu na nowo ustala v0 (value-residual). K w config ckpt ("loop"); + --init-from sprawdza zgodnosc K. Domyslnie K=1 = r11 bit w bit (config ckpt bez klucza "loop"). +r13 (Hart, 11.10, kampania 2 Slayer9, Latarnik 11:40Z): hybryda GatedDeltaNet2 (flash-linear-attention 0.5.2) + uwaga. + --mixer-layout STR litera na blok: A = uwaga jak r12, G = fla.layers.gdn2.GatedDeltaNet2 (hidden d, head_dim + --gdn-head-dim, num_heads d/head_dim, expand_v 1, short conv 4, mode chunk) w miejsce uwagi; FFN, + normy i reszta bloku bez zmian. Domyslnie brak = same A = r12 bit w bit (config bez nowych kluczy). + Value-residual: v0 ustala PIERWSZY blok A (w r12 blok 0). Wagi G inicjalizuje fla (bez _init GPT). + Optymalizator: w blokach G Muon tylko q/k/v/o_proj; reszta G (konwolucje, bramki, A_log, dt_bias, + normy) -> AdamW (karta kampanii 2: czesc „budowy Q-ARCH”, ustalona przed biegiem). + --attn-bias 0|1 bias w qkv/proj uwagi (domyslnie 1 jak r12; Qielbas M2 = 0). + --rope-style interleaved (r12) | half (rotate-half jak Qielbas M2; baza z --rope-theta). + Import fla tylko przy G w ukladzie (PYTHONPATH z fla-lib), wiec same A dzialaja bez fla. +r14 (Hart, 11.10, kampania 3 Slayer9: KD w pretreningu, Arek 15:56Z): destylacja od nauczyciela na zywo, domyslnie WYLACZONA + (bez --kd-teacher = r13 bit w bit). + --kd-teacher CKPT --kd-teacher-sha HEX64 ckpt nauczyciela (r12/r13, ten sam trener i vocab co uczen; sha pelne, inaczej + STOP). Nauczyciel budowany z config ckpt, eval, bez gradientu, forward w tym samym autocast co uczen + (bf16), w kazdym kroku na tym samym batchu. Budowa nauczyciela nie zuzywa globalnego RNG (fork_rng). + --kd-alpha A strata = (1-A)*CE + A*T^2*KL(p_naucz || p_ucz), obie srednie po B*T pozycjach; KL po pelnym vocab, + w fp32, z logitow /T (uczen po --logit-cap, jak w CE). Z --loss-chunk liczone tymi samymi kawalkami. + --kd-temp T temperatura (domyslnie 1.0). + Walidacja (VAL loss) = sama CE jak w r13. W logu kroku dodatkowo ce (sama CE). W ckpt klucz "kd" (poza "config", + wiec valport/--init-from bez zmian). +r15 (Hart, 11.10, Arek 21:33Z „eksperymenty do skutku”, przeglad arXiv #4): faktoryzowane osadzenia, domyslnie WYLACZONE + (bez --emb-dim = r14 bit w bit). + --emb-dim E tok = Embedding(V, E) -> emb_up Linear(E, d); glowica: emb_down Linear(d, E) -> head Linear(E, V) + zwiazana z tok (te same wagi V x E). P osadzen V*E + 2*E*d zamiast V*d. emb_up/emb_down: init + N(0, 1/E) i N(0, 1/d) (skala strumienia jak bez faktoryzacji), optymalizator jak macierze ukryte (Muon). + W config ckpt "emb_dim" tylko przy E > 0. +r16 (Hart, 11.10, kolejka nocna pkt 4 Latarnika: biegi ~V1 po 1 na karte): akumulacja gradientu, domyslnie WYLACZONA + (bez --grad-accum = r15 bit w bit). + --grad-accum K wiersze ranku (batch / world) dzielone na K rownych, ciaglych mikrobatchy; backward (strata_j / K), + pod DDP no_sync poza ostatnim. Te same wiersze co bez akumulacji (sampler globalny), wiec krok = + krok bez akumulacji z dokladnoscia do kolejnosci sumowania. Strata logu = srednia mikrobatchy. +r17 (Hart, 12.10, karta Q-REPRO Mierniczego a92e717c: replika przepisu Qielbas M2): opcje optymalizatora z ich + `pretraining-report.json`, wszystkie domyslnie WYLACZONE (bez nich = r16 bit w bit). + --muon-scale match_rms_adamw skala kroku Muon jak torch.optim.Muon(adjust_lr_fn="match_rms_adamw"): + 0.2 * sqrt(max(wiersze, kolumny)) zamiast max(1, wiersze/kolumny)^0.5; z --muon-lr = --lr + (ich „peak lr=0.001” wspolne dla Muon i AdamW). + --muon-wd W decoupled weight decay na parametrach Muon (p *= 1 - lr*W, lr przed skalowaniem, jak torch); + domyslnie 0 (jak dotad: WD tylko na AdamW-aux). + --tf32 0 TF32 wylaczone w matmul/cudnn (ich „TF32 disabled”); domyslnie 1 (jak dotad). + Harmonogram cosine -> min-lr, warmup, swiezy optymalizator przy --init-from: bez zmian (juz byly). +r18 (Hart, 13.10, ramie TC-arch po krzywej rozmiaru). Fragmenty XSA, partial RoPE, ReLU^2 i zero-init ADAPTED FROM + kurcontko/Tiny-Chick-10M@60d32c4d tinylm_arch.py (sha256 4876824d...), Apache License 2.0 + (https://www.apache.org/licenses/LICENSE-2.0); zmiany: wpiecie w Block/GPT r17, flagi CLI, walidacja, config ckpt. + wszystkie domyslnie WYLACZONE (bez nich = r17 bit w bit, config ckpt bez nowych kluczy): + --xsa 1 po uwadze z wyjscia kazdej glowy usuwana skladowa wzdluz jej (znormalizowanego) wektora value tokenu: + y -= (y . v^) v^ (v = to, co weszlo do SDPA, czyli po value-residual). + --rope-dims N RoPE tylko na pierwszych N wymiarach glowy (parzyste, 0 < N < head_dim; czestotliwosci liczone z N), + reszta wymiarow bez pozycji; 0 = wszystkie (jak r17). Styl interleaved/half bez zmian, w obrebie N. + --ffn relu2 MLP down(relu(up(x))^2) bez biasu, hidden = round(--ffn-mult * d) (Tiny Chick: mult 4); gelu/swiglu bez zmian. + --zero-init 1 po inicjalizacji zerowane wagi wyjscia uwagi (proj, bloki A) i wyjscia MLP (down / ostatni Linear, kazdy blok); + bloki G zostaja z inicjalizacja fla. + --init-from i nauczyciel KD: config ckpt bez kluczy r18 = wartosci domyslne; rozjazd xsa/rope_dims/ffn z argumentami = STOP. +r19 (Hart, 13.10, karta TC-REPLIKA): backend --arch tinylm = model z ORYGINALNEGO pliku kurcontko/Tiny-Chick-10M@60d32c4d + tinylm_arch.py (sha256 4876824de8c697a6..., Apache License 2.0, uzyty bez zmian, ladowany z --tinylm-file i sprawdzany po + sha; NOTICE: "derived from kurcontko/Tiny-Chick-10M"). Bez --arch tinylm = r18 bit w bit (config ckpt bez nowych kluczy). + Z --arch tinylm: GPT trzyma TinyLM jako self.tl (klucze state_dict "tl."); forward zwraca (logity, CE) jak GPT. + TinyConfig z argumentow: vocab, n_layer, n_embd, n_head, --kv-heads, --ffn relu2|swiglu, --ffn-mult, --qk-norm, + --rope-dims, --rope-theta, --xsa, --zero-init, --norm-eps, --block. Optymalizator Muon: tl.emb -> AdamW (jak tok/head), + skalarny gain QK-norm (ndim 3) -> AdamW, reszta 2D -> Muon. Niezgodne z tinylm (STOP): --loop > 1, --mixer-layout, + --emb-dim, --value-residual, --kd-teacher, --act-ckpt, --loss-chunk, --logit-cap, --z-loss, --pos learned, --rope-style + interleaved, --norm layernorm, --attn-bias 1. +r20 (Hart, 13.10): torch.load ckpt przy --resume i --init-from z weights_only=True (uwaga Wartownika do r18 l. 832: + weights_only=False wykonuje dowolny pickle). Ckpt trenera (model / opt / step / config / queue) laduje sie tak bez zmian + (sprawdzone na ba223162). Zapis ckpt po ostatnim kroku biegu także bez --queue, gdy end_step % ckpt_every != 0 + (r18/r19: tylko z --queue; CPT k9 13.10 skonczyl z ckpt 1140 zamiast 1145). Poza tym r20 = r19 bit w bit. +r21 (Monter dla Harta, 14.10, Axolote maggi-uxz6, przepis_axolote_v0 2.2): --train-only REGEX, domyslnie WYLACZONE (bez + flagi = r20 bit w bit, ckpt bez nowych kluczy). Parametry, ktorych nazwa nie pasuje do REGEX (re.search), maja + requires_grad False i nie trafiaja do optymalizatora (muon juz je pomijal, adamw filtruje), wiec zostaja bit w bit. + Tylko z --init-from; regex bez trafien albo chwytajacy osadzenia / tied head = STOP. Wzor: ft_arc11.py --train-only. + W ckpt klucz "train_only" {regex, n_trainable} poza "config" (jak "kd"), wiec --init-from i valport bez zmian. +r22 (Hart, 17.10, maggi-v38t / karta maggi-jyoy): --imha-t T, domyslnie 0 = WYLACZONE (r21 bit w bit, ckpt bez nowych kluczy). + T >= 1: podwarstwa uwagi blokow A = IMHA (Divakar i Priyadarshini, TAI 2(1) 2026, doi 10.53941/tai.2026.100014, Algorytm 1): + h0 = ln1(x); T iteracji tymi samymi qkv / proj: A = softmax(QK^T/sqrt(dk) + M) jawnie (fp32), z = AV; entropia wiersza + per glowa H_i znormalizowana przez log(i+1) (H_0 = 0); w_i = exp(-H_i/tau) / srednia prefiksu exp(-H_k/tau), k <= i; + z_i *= w_i; z = proj([z_h]); g_i = srednia prefiksu z; h += sigmoid(W_a[z;g]) * relu(W_f[z;g]); wyjscie x += h_T - h0. + Nowe parametry na blok: imha_a, imha_f (Linear 2d -> d, bez biasu: w artykule W in R^{2d x d}) i imha_log_tau + (tau = exp(log_tau) > 0, init tau = 1; artykul: "learnable temperature per layer, initialised at 1"). + Decyzje poza artykulem (nasz blok ma wiecej niz MHA z artykulu): qk-norm i RoPE stosowane w kazdej iteracji jak w r21; + value residual: v0 ustala blok pierwszy w iteracji 0, kolejne iteracje i bloki mieszaja v z v0 jak w r21. + STOP: --imha-t z --xsa, --mixer-layout (bloki G), --arch tinylm. Atrybut testowy cfg.imha_plain (tylko test_r22.py): + entropia, kontekst globalny i bramka wylaczone -> przy T = 1 ma byc rowne SDPA r21 (pozytywna kontrola z artykulu, 3.3.7). + Koszt: jawna macierz uwagi B x H x N x N zamiast SDPA (wolniej i wiecej pamieci) -> rownowazenie kosztu z tok/s z pomiaru. +r23 (Hart, 20.10, karta dzien dowodow 400M, Mierniczy 19:26Z): --kv-heads dla --arch gpt (GQA), domyslnie 0 = r22 bit w bit + (te same nazwy/ksztalty wag, ten sam RNG inicjalizacji, te same wyjscia, config ckpt bez nowych kluczy; K = n_head = jak 0). + K > 0: w blokach A qkv = jeden Linear d -> d + 2*K*hd (bias wg --attn-bias, Muon jak dotad, bo 2D); q ma n_head glow, + k i v po K glow. QK-norm i RoPE na k z K glowami, value residual (v0, vr_lambda) na v z K glowami, potem k, v powielone + repeat_interleave na n_head glow (grupa = n_head / K kolejnych glow q) przed SDPA; XSA na v po powieleniu (per glowa q). + Bloki G bez zmian. W config ckpt "kv_heads" tylko przy K > 0; --init-from sprawdza kv_heads (brak klucza = 0), nauczyciel + KD czyta go z configu. STOP: n_head % K != 0, K > n_head, K < 0, --kv-heads z --imha-t. + --data-seed S (karta dowodów v0.1 pkt 2, Mierniczy 20:56Z): ziarno kolejności danych (generator prefetchera i samplery + epoch / queue) osobno od --seed (inicjalizacja modelu); domyślnie brak = --seed, czyli r22 bit w bit. Ramiona R × 3 + różnią się --seed przy stałym --data-seed. W config ckpt "data_seed" tylko, gdy podane; --resume z innym = STOP. +""" +import argparse +import hashlib +import json +import math +import os +import re +import time +import queue +import threading +from contextlib import nullcontext + +import numpy as np +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.checkpoint import checkpoint +from torch.nn.parallel import DistributedDataParallel as DDP + + +def get_args(): + p = argparse.ArgumentParser() + p.add_argument("--data-dir", default=".") + p.add_argument("--out-dir", default=None) + p.add_argument("--run-id", default="gpt25m_run1") + p.add_argument("--steps", type=int, default=30000) + p.add_argument("--batch", type=int, default=32) + p.add_argument("--block", type=int, default=256) + p.add_argument("--n-layer", type=int, default=8) + p.add_argument("--n-embd", type=int, default=512) + p.add_argument("--n-head", type=int, default=8) + p.add_argument("--lr", type=float, default=6e-4) + p.add_argument("--min-lr", type=float, default=6e-5) + p.add_argument("--warmup", type=int, default=200) + p.add_argument("--schedule", choices=["cosine", "wsd"], default="cosine", + help="cosine (default) | wsd (warmup-stable-decay: constant peak then 1-sqrt decay, JugnuLM ARC-closer)") + p.add_argument("--decay-frac", type=float, default=0.2, + help="WSD: fraction of steps in the final decay phase (default 0.2)") + p.add_argument("--rewarm-steps", type=int, default=0, + help="przy --resume: liniowe przejscie LR od wartosci z ckpt do harmonogramu przez N krokow (0=off)") + p.add_argument("--wd", type=float, default=0.1) + p.add_argument("--grad-clip", type=float, default=1.0) + p.add_argument("--log-every", type=int, default=50) + p.add_argument("--eval-every", type=int, default=500) + p.add_argument("--eval-iters", type=int, default=50) + p.add_argument("--ckpt-every", type=int, default=1000) + p.add_argument("--seed", type=int, default=1337) + p.add_argument("--data-seed", type=int, default=None, + help="r23: ziarno kolejności danych (domyślnie = --seed); --seed zostaje ziarnem inicjalizacji modelu") + p.add_argument("--sampler", choices=["random", "epoch"], default="random", + help="random (r5: okna ze zwracaniem, ~75%% korpusu przy 1.4 ep.) | epoch (r6: kazde okno raz na epoke, " + "per-epoka permutacja + losowe przesuniecie siatki; resume dokladny bez fast-forwardu)") + p.add_argument("--resume", action="store_true") + p.add_argument("--init-from", default=None, + help="r10 (Wzrost): start z wag innego ckpt (np. z grow_depth.py), krok 0 i swiezy optymalizator; " + "ignorowane, gdy --resume wznawia wlasny ckpt.pt tego biegu") + p.add_argument("--mixer-layout", default=None, + help="r13: litera na blok (A = uwaga, G = GatedDeltaNet2), dlugosc = --n-layer; brak = same A (r12)") + p.add_argument("--gdn-head-dim", type=int, default=64, help="r13: head_dim GatedDeltaNet2") + p.add_argument("--attn-bias", type=int, choices=(0, 1), default=1, help="r13: bias w qkv/proj uwagi (1 = r12)") + p.add_argument("--rope-style", choices=("interleaved", "half"), default="interleaved", + help="r13: interleaved (r12) | half (rotate-half)") + p.add_argument("--data-offset-steps", type=int, default=0, + help="r10 (Wzrost, tylko --sampler epoch): dane od kroku N strumienia, np. dalszy ciag po ckpt " + "mniejszego modelu (ten sam batch i seed), bez powtarzania widzianych okien") + p.add_argument("--data-dir-2", default=None, + help="r10 (Wzrost, tylko --sampler epoch): od kroku strumienia --switch-step okna z DATA_DIR_2/train.bin " + "(np. swieze dokumenty rozlaczne z base), wlasna permutacja epokowa liczona od --switch-step") + p.add_argument("--switch-step", type=int, default=0, help="krok strumienia przejscia na --data-dir-2") + p.add_argument("--batch-sha", default=None, + help="r10 (Wzrost): bez treningu; sha256 paczek (x+y) w podanych krokach biegu, np. 0,24413,24414, " + "z tego samego Prefetchera co trening (kontrola strumienia), potem koniec") + p.add_argument("--synthetic", action="store_true", help="smoke throughput na losowym uint8 (bez danych)") + p.add_argument("--vocab", type=int, default=256) + p.add_argument("--dtype", default="uint8", help="bin dtype: uint8 (byte) | uint16 (BPE)") + p.add_argument("--optimizer", choices=["adamw", "muon"], default="adamw", + help="adamw (default, backward-compat) | muon (Newton-Schulz ortho dla 2D-weights + AdamW dla reszty)") + p.add_argument("--muon-lr", type=float, default=0.02, + help="peak LR dla Muon (macierzowe params); AdamW-aux uzywa --lr. Muon skalowany ta sama cosine-schedule co AdamW przez lr_mult=muon_lr/lr") + p.add_argument("--norm", choices=["layernorm", "rmsnorm"], default="layernorm", + help="layernorm (default, backward-compat) | rmsnorm (Qwen3-style, fp32-compute)") + p.add_argument("--norm-eps", type=float, default=1e-6) + p.add_argument("--pos", choices=["learned", "rope"], default="learned", + help="learned (default) | rope (parameter-free RoPE na q,k; usuwa learned pos-embedding)") + p.add_argument("--rope-theta", type=float, default=100000.0, + help="RoPE theta; top-3-board (JugnuLM/GPT-X2) uzywaja 100000 (nie-default-10K)") + p.add_argument("--ffn", choices=["gelu", "swiglu", "relu2"], default="gelu", + help="gelu (default 4x MLP) | swiglu (Qwen3 gated-MLP, hidden=--ffn-mult*d) | relu2 (r18: down(relu(up)^2), bez biasu)") + p.add_argument("--ffn-mult", type=float, default=2.667, + help="mnoznik hidden dla swiglu (~param-parity z 4x-gelu przy 8/3)") + p.add_argument("--value-residual", action="store_true", + help="ResFormer value-residuals: v_l += lambda_l*v0 (lambda init 0), ARC-targeted") + p.add_argument("--qk-norm", action="store_true", + help="Qwen3 QK-Norm: RMSNorm per-head na Q,K przed-attention (stabilnosc z Muon/high-LR)") + p.add_argument("--events-jsonl", default=None, + help="jesli podane: emituj events.jsonl (fabryka-track sidecar-format: update/evaluation/checkpoint/end)") + p.add_argument("--compile", action="store_true", + help="torch.compile model (2-3x throughput; state_dict zapisywany bez _orig_mod prefix via raw_model)") + p.add_argument("--logit-cap", type=float, default=0.0, + help="Gemma-style logit soft-cap: logits=cap*tanh(logits/cap); 0=off (prob. 30)") + p.add_argument("--z-loss", type=float, default=0.0, + help="z-loss coef: +coef*mean(logsumexp(logits)^2), stabilizuje logity; 0=off (prob. 1e-4)") + p.add_argument("--act-ckpt", action="store_true", + help="r11: przeliczanie aktywacji po bloku w treningu (pamiec ~ wejscia blokow zamiast wszystkich aktywacji)") + p.add_argument("--loss-chunk", type=int, default=0, + help="r11: strata kawalkami po N pozycji, bez pelnych logitow [B,T,V]; 0 = off (r10)") + p.add_argument("--loop", type=int, default=1, + help="r12: ile razy przejsc przez n_layer wspoldzielonych blokow (1 = bez wspoldzielenia, jak r11)") + p.add_argument("--queue", default=None, + help="r7: manifest kolejki shardow (JSON: steps_per_shard, shards{id:{lo,hi,sha256,source}}, rotations[[id..]..]). " + "Slot j = kroki [j*S,(j+1)*S) na shardzie schedule[j]; schedule = rotacje po kolei. Bez flagi = r6 bit w bit.") + p.add_argument("--queue-one-slot", action="store_true", + help="r7: trenuj tylko do konca biezacego slotu i zapisz ckpt (krok nadzorcy kolejki decyduje o nastepnym)") + p.add_argument("--kd-teacher", default=None, help="r14: ckpt nauczyciela KD (r12/r13); bez = r13 bit w bit") + p.add_argument("--kd-teacher-sha", default=None, help="r14: pelne sha256 ckpt nauczyciela (wymagane z --kd-teacher)") + p.add_argument("--kd-alpha", type=float, default=0.0, help="r14: waga KL w (1-A)*CE + A*T^2*KL; 0 < A <= 1 z nauczycielem") + p.add_argument("--kd-temp", type=float, default=1.0, help="r14: temperatura KD") + p.add_argument("--emb-dim", type=int, default=0, help="r15: faktoryzacja osadzen V x E + E x d (0 = wylaczone, r14)") + p.add_argument("--grad-accum", type=int, default=1, help="r16: mikrobatche na krok (1 = wylaczone, r15)") + p.add_argument("--muon-scale", choices=["sqrt_ratio", "match_rms_adamw"], default="sqrt_ratio", + help="r17: skala kroku Muon (sqrt_ratio = jak dotad; match_rms_adamw = torch.optim.Muon)") + p.add_argument("--muon-wd", type=float, default=0.0, help="r17: decoupled WD na parametrach Muon (0 = jak dotad)") + p.add_argument("--tf32", type=int, choices=[0, 1], default=1, help="r17: 0 = TF32 wylaczone (jak Qielbas M2)") + p.add_argument("--xsa", type=int, choices=[0, 1], default=0, help="r18: 1 = z wyjscia glowy usuwana skladowa wzdluz wlasnego v") + p.add_argument("--rope-dims", type=int, default=0, help="r18: RoPE na pierwszych N wymiarach glowy (0 = wszystkie, r17)") + p.add_argument("--zero-init", type=int, choices=[0, 1], default=0, help="r18: 1 = zerowe wagi proj uwagi i wyjscia MLP") + p.add_argument("--arch", choices=["gpt", "tinylm"], default="gpt", help="r19: tinylm = model z pliku Tiny Chick (--tinylm-file)") + p.add_argument("--tinylm-file", default=None, help="r19: tinylm_arch.py z kurcontko/Tiny-Chick-10M@60d32c4d (sha 4876824d...)") + p.add_argument("--kv-heads", type=int, default=0, + help="r19 tinylm / r23 gpt: liczba glow KV (GQA, dzielnik --n-head); 0 = --n-head (MHA)") + p.add_argument("--train-only", default=None, + help="r21: regex nazw uczonych parametrow; reszta requires_grad False i poza optymalizatorem; tylko z --init-from") + p.add_argument("--imha-t", type=int, default=0, + help="r22: IMHA z T iteracjami w blokach uwagi (0 = zwykla uwaga, r21 bit w bit); artykul: T = 2") + return p.parse_args() + + +# ---- Muon (Keller Jordan) -------------------------------------------------- +# Ref: https://github.com/KellerJordan/Muon (modded-nanogpt). Muon = momentum +# SGD, ale update ortogonalizowany przez ~5 krokow iteracji Newtona-Schulza +# (przyblizona ortogonalizacja macierzy gradientu). Stosowany TYLKO do +# macierzowych ukrytych wag (ndim>=2: qkv/proj/mlp). Embeddingi (tok/pos), head +# (tied), LayerNorm-gains i biasy ida do zwyklego AdamW. +def zeropower_via_newtonschulz5(G, steps=5): + """Ortogonalizacja macierzy G przez quintic Newton-Schulz (bf16). Zwraca + macierz ~ U V^T z SVD(G)=U S V^T. Wspolczynniki (a,b,c) z impl. Kellera.""" + assert G.ndim == 2 + a, b, c = (3.4445, -4.7750, 2.0315) + X = G.bfloat16() + transposed = G.size(0) > G.size(1) + if transposed: + X = X.T + X = X / (X.norm() + 1e-7) + for _ in range(steps): + A = X @ X.T + B = b * A + c * (A @ A) + X = a * X + B @ X + if transposed: + X = X.T + return X + + +class Muon(torch.optim.Optimizer): + """Momentum-SGD z ortogonalizowanym update. weight_decay domyslnie 0 (Muon-params + czysto; WD trzymamy na AdamW-aux). lr_mult pozwala petli lr-schedule skalowac + Muon proporcjonalnie do AdamW.""" + def __init__(self, params, lr=0.02, lr_mult=1.0, momentum=0.95, nesterov=True, + ns_steps=5, weight_decay=0.0, scale_mode="sqrt_ratio"): + defaults = dict(lr=lr, lr_mult=lr_mult, momentum=momentum, nesterov=nesterov, + ns_steps=ns_steps, weight_decay=weight_decay, scale_mode=scale_mode) + super().__init__(params, defaults) + + @torch.no_grad() + def step(self, closure=None): + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + for group in self.param_groups: + lr = group["lr"]; momentum = group["momentum"]; wd = group["weight_decay"] + for p in group["params"]: + g = p.grad + if g is None: + continue + if g.ndim > 2: + g = g.reshape(g.size(0), -1) + state = self.state[p] + if "momentum_buffer" not in state: + state["momentum_buffer"] = torch.zeros_like(g) + buf = state["momentum_buffer"] + buf.mul_(momentum).add_(g) + g = g.add(buf, alpha=momentum) if group["nesterov"] else buf + u = zeropower_via_newtonschulz5(g, steps=group["ns_steps"]) + if wd != 0: + p.mul_(1 - lr * wd) + # scale ~ sqrt(fan_out/fan_in): zrownuje RMS update niezaleznie od ksztaltu + # r17 match_rms_adamw: 0.2*sqrt(max(A,B)) jak torch.optim.Muon (RMS kroku ~ AdamW) + if group.get("scale_mode", "sqrt_ratio") == "match_rms_adamw": + scale = 0.2 * max(g.size(0), g.size(1)) ** 0.5 + else: + scale = max(1.0, p.size(0) / p.size(1)) ** 0.5 + p.add_(u.reshape(p.shape).to(p.dtype), alpha=-lr * scale) + return loss + + +class MuonWithAuxAdam: + """Kontener: Muon dla macierzowych ukrytych wag + AdamW dla reszty. Wystawia + param_groups/step/zero_grad/state_dict tak, by petla treningowa dzialala bez zmian.""" + def __init__(self, muon, adamw): + self.muon = muon + self.adamw = adamw + + @property + def param_groups(self): + return self.muon.param_groups + self.adamw.param_groups + + @property + def state(self): + return {**self.muon.state, **self.adamw.state} + + def step(self, closure=None): + self.muon.step() + self.adamw.step() + + def zero_grad(self, set_to_none=True): + self.muon.zero_grad(set_to_none=set_to_none) + self.adamw.zero_grad(set_to_none=set_to_none) + + def state_dict(self): + return {"muon": self.muon.state_dict(), "adamw": self.adamw.state_dict()} + + def load_state_dict(self, sd): + self.muon.load_state_dict(sd["muon"]) + self.adamw.load_state_dict(sd["adamw"]) + + +def build_optimizer(a, model): + """--optimizer adamw -> DOKLADNIE poprzedni AdamW (backward-compat). + --optimizer muon -> Muon(2D-hidden) + AdamW(embeddingi/head/norm/bias).""" + if a.optimizer == "adamw": + return torch.optim.AdamW([p for p in model.parameters() if p.requires_grad], lr=a.lr, weight_decay=a.wd, + betas=(0.9, 0.95)) # r21: bez zamrozonych (--train-only) + muon_params, adamw_params, seen = [], [], set() + for name, p in model.named_parameters(): + if not p.requires_grad or id(p) in seen: + continue + seen.add(id(p)) + is_embed_or_head = name.startswith(("tok.", "pos.", "head.", "tl.emb.")) # r19: osadzenia TinyLM (tied head) jak tok + tl_gain = name.startswith("tl.") and name.endswith(".attn.gain") # r19: skalar QK-norm na glowe [H,1,1] -> AdamW + # r13: w blokach G do Muon tylko kwadratowe projekcje q/k/v/o; konwolucje (3D), bramki niskiego rzedu, + # b/w_proj, A_log, dt_bias, normy -> AdamW. Bez blokow G warunek jest pusty (r12 bit w bit). + gdn_aux = ".gdn." in name and not name.endswith(("q_proj.weight", "k_proj.weight", "v_proj.weight", "o_proj.weight")) + if p.ndim >= 2 and not is_embed_or_head and not gdn_aux and not tl_gain: + muon_params.append(p) + else: + adamw_params.append(p) + lr_mult = a.muon_lr / a.lr if a.lr > 0 else 1.0 + muon = Muon(muon_params, lr=a.muon_lr, lr_mult=lr_mult, weight_decay=getattr(a, "muon_wd", 0.0), + scale_mode=getattr(a, "muon_scale", "sqrt_ratio")) + adamw = torch.optim.AdamW(adamw_params, lr=a.lr, weight_decay=a.wd, betas=(0.9, 0.95)) + return MuonWithAuxAdam(muon, adamw) + + +class RMSNorm(nn.Module): + """Qwen3-style RMSNorm (fp32-compute dla stabilnosci). 1D weight -> AdamW w split-Muon.""" + def __init__(self, d, eps=1e-6): + super().__init__() + self.weight = nn.Parameter(torch.ones(d)) + self.eps = eps + + def forward(self, x): + return x * torch.rsqrt(x.float().pow(2).mean(-1, keepdim=True) + self.eps).to(x.dtype) * self.weight + + +def make_norm(d, cfg): + return RMSNorm(d, cfg.norm_eps) if cfg.norm == "rmsnorm" else nn.LayerNorm(d) + + +def apply_rope(x, base=100000.0, style="interleaved", nrot=0): + """Parameter-free RoPE na [B,H,T,D] (interleaved-conv, port z qwen_model.py). Train==eval + MUSZA uzywac tej samej konwencji (self-contained eval -> spojne). + r13: style="half" = rotate-half (pary i, i+D/2), jak Qielbas M2; "interleaved" = r12 bez zmian. + r18: nrot (0 < nrot < D) = obrot tylko pierwszych nrot wymiarow (czestotliwosci z nrot), reszta bez zmian; 0 = calosc.""" + if 0 < nrot < x.size(-1): + return torch.cat([apply_rope(x[..., :nrot], base, style), x[..., nrot:]], dim=-1) + _, _, T, dim = x.shape + pos = torch.arange(T, device=x.device, dtype=torch.float32) + freq = 1.0 / (base ** (torch.arange(0, dim, 2, device=x.device, dtype=torch.float32) / dim)) + ang = torch.outer(pos, freq) + if style == "half": + ang = torch.cat([ang, ang], dim=-1) + cos, sin = ang.cos().to(x.dtype)[None, None], ang.sin().to(x.dtype)[None, None] + a, b = x.chunk(2, dim=-1) + return x * cos + torch.cat([-b, a], dim=-1) * sin + cos, sin = ang.cos().to(x.dtype)[None, None], ang.sin().to(x.dtype)[None, None] + even, odd = x[..., ::2], x[..., 1::2] + return torch.stack((even * cos - odd * sin, even * sin + odd * cos), dim=-1).flatten(-2) + + +class SwiGLU(nn.Module): + """Qwen3 gated-MLP: down(silu(gate(x))*up(x)). 3x 2D bez-bias -> wszystkie do Muon.""" + def __init__(self, d, hidden): + super().__init__() + self.gate = nn.Linear(d, hidden, bias=False) + self.up = nn.Linear(d, hidden, bias=False) + self.down = nn.Linear(hidden, d, bias=False) + + def forward(self, x): + return self.down(F.silu(self.gate(x)) * self.up(x)) + + +class ReLU2(nn.Module): + """r18 (Tiny Chick): down(relu(up(x))^2), 2x 2D bez biasu -> Muon.""" + def __init__(self, d, hidden): + super().__init__() + self.up = nn.Linear(d, hidden, bias=False) + self.down = nn.Linear(hidden, d, bias=False) + + def forward(self, x): + return self.down(F.relu(self.up(x)).square()) + + +class Block(nn.Module): + def __init__(self, d, nh, block, cfg, is_first=False, mixer="A", layer_idx=0): + super().__init__() + self.ln1 = make_norm(d, cfg) + self.ln2 = make_norm(d, cfg) + self.mixer = mixer + if mixer == "G": # r13: GatedDeltaNet2 w miejsce uwagi (fla 0.5.2); inicjalizacja fla, nie _init GPT + from fla.layers.gdn2 import GatedDeltaNet2 + hd = getattr(cfg, "gdn_head_dim", 64) + if d % hd: + raise SystemExit(f"STOP: n_embd {d} nie dzieli sie przez --gdn-head-dim {hd}") + self.gdn = GatedDeltaNet2(hidden_size=d, head_dim=hd, num_heads=d // hd, expand_v=1.0, + use_short_conv=True, mode="chunk", layer_idx=layer_idx) + for sm in self.gdn.modules(): + sm._r13_fla_init = True + else: + bias = bool(getattr(cfg, "attn_bias", 1)) + kvh = int(getattr(cfg, "kv_heads", 0) or 0) # r23: GQA (0 albo n_head = MHA, qkv d -> 3d jak r22) + self.nkv = kvh if 0 < kvh < nh else nh + self.kvd = self.nkv * (d // nh) + self.qkv = nn.Linear(d, d + 2 * self.kvd, bias=bias) + self.proj = nn.Linear(d, d, bias=bias) + if cfg.ffn == "swiglu": + self.mlp = SwiGLU(d, int(round(cfg.ffn_mult * d))) + elif cfg.ffn == "relu2": + self.mlp = ReLU2(d, int(round(cfg.ffn_mult * d))) + else: + self.mlp = nn.Sequential(nn.Linear(d, 4 * d), nn.GELU(), nn.Linear(4 * d, d)) + self.nh = nh + self.d = d + self.cfg = cfg + self.is_first = is_first + if mixer == "A" and cfg.value_residual and not is_first: + self.vr_lambda = nn.Parameter(torch.zeros(1)) + if mixer == "A" and cfg.qk_norm: + hd = d // nh + self.q_norm = RMSNorm(hd, cfg.norm_eps) + self.k_norm = RMSNorm(hd, cfg.norm_eps) + self.imha_t = int(getattr(cfg, "imha_t", 0) or 0) if mixer == "A" else 0 + if self.imha_t and self.nkv != nh: + raise SystemExit("STOP --imha-t: niezgodne z --kv-heads (GQA)") + if self.imha_t: # r22: bramka i kandydat IMHA (W_a, W_f in R^{2d x d}) + temperatura per warstwa + self.imha_a = nn.Linear(2 * d, d, bias=False) + self.imha_f = nn.Linear(2 * d, d, bias=False) + self.imha_log_tau = nn.Parameter(torch.zeros(1)) + + def _qkv(self, h, v0, set_v0): + B, T, D = h.size() + q, k, v = self.qkv(h).split(self.d, dim=2) + hd = D // self.nh + q = q.view(B, T, self.nh, hd).transpose(1, 2) + k = k.view(B, T, self.nh, hd).transpose(1, 2) + v = v.view(B, T, self.nh, hd).transpose(1, 2) + if self.cfg.qk_norm: + q = self.q_norm(q) + k = self.k_norm(k) + if self.cfg.pos == "rope": + style = getattr(self.cfg, "rope_style", "interleaved") + nrot = int(getattr(self.cfg, "rope_dims", 0) or 0) + q = apply_rope(q, self.cfg.rope_theta, style, nrot) + k = apply_rope(k, self.cfg.rope_theta, style, nrot) + if self.cfg.value_residual: + if self.is_first: # blok 0 nie ma vr_lambda (r21): v0 = v z iteracji t = 0, w t >= 1 v bez mieszania + if set_v0: + v0 = v + else: + v = v + self.vr_lambda * v0 + return q, k, v, v0 + + def _imha(self, x, v0): + """r22: IMHA (Algorytm 1); zwraca (h_T - h_0, v0). Wszystkie sumy po k <= i (prefiks), wiec przyczynowo.""" + B, T, D = x.size() + hd = D // self.nh + h0 = self.ln1(x) + h = h0 + pos = torch.arange(T, device=x.device) + cnt = (pos + 1).to(torch.float32) # i + 1 + logn = torch.log(cnt).clamp_min(1e-12) # log(i+1); i = 0 -> H_0 = 0 nizej + mask = torch.ones(T, T, device=x.device, dtype=torch.bool).tril() + plain = bool(getattr(self.cfg, "imha_plain", False)) + tau = self.imha_log_tau.exp() + for t in range(self.imha_t): + q, k, v, v0 = self._qkv(h, v0, set_v0=(t == 0)) + s = (q.float() @ k.float().transpose(-2, -1)) * (hd ** -0.5) + A = torch.softmax(s.masked_fill(~mask, float("-inf")), dim=-1) # [B,H,T,T] fp32 + z = (A.to(v.dtype) @ v) # [B,H,T,hd] + if not plain: + ent = -(A * torch.log(A.clamp_min(1e-30))).sum(-1) # 0 log 0 = 0 (A = 0 -> 0 * log(1e-30) = 0) + hn = torch.where(pos == 0, torch.zeros_like(ent), ent / logn) + e = torch.exp(-hn / tau) + w = e / (e.cumsum(-1) / cnt) # srednia prefiksu, E[w] ~ 1 + z = z * w.unsqueeze(-1).to(z.dtype) + z = self.proj(z.transpose(1, 2).contiguous().view(B, T, D)) + if plain: + h = h + z + continue + g = z.cumsum(1) / cnt.view(1, T, 1).to(z.dtype) + u = torch.cat([z, g], dim=-1) + h = h + torch.sigmoid(self.imha_a(u)) * F.relu(self.imha_f(u)) + return h - h0, v0 + + def forward(self, x, v0=None): + B, T, D = x.size() + if self.imha_t: + dx, v0 = self._imha(x, v0) + x = x + dx + x = x + self.mlp(self.ln2(x)) + return x, v0 + h = self.ln1(x) + if self.mixer == "G": + x = x + self.gdn(h, use_cache=False)[0] + x = x + self.mlp(self.ln2(x)) + return x, v0 + q, k, v = self.qkv(h).split([self.d, self.kvd, self.kvd], dim=2) + hd = D // self.nh + q = q.view(B, T, self.nh, hd).transpose(1, 2) + k = k.view(B, T, self.nkv, hd).transpose(1, 2) + v = v.view(B, T, self.nkv, hd).transpose(1, 2) + if self.cfg.qk_norm: + q = self.q_norm(q) + k = self.k_norm(k) + if self.cfg.pos == "rope": + style = getattr(self.cfg, "rope_style", "interleaved") + nrot = int(getattr(self.cfg, "rope_dims", 0) or 0) + q = apply_rope(q, self.cfg.rope_theta, style, nrot) + k = apply_rope(k, self.cfg.rope_theta, style, nrot) + if self.cfg.value_residual: + if self.is_first: + v0 = v + else: + v = v + self.vr_lambda * v0 + if self.nkv != self.nh: # r23: GQA, glowy KV powielone na grupy glow q (po RoPE / QK-norm / value-residual) + k = k.repeat_interleave(self.nh // self.nkv, dim=1) + v = v.repeat_interleave(self.nh // self.nkv, dim=1) + y = F.scaled_dot_product_attention(q, k, v, is_causal=True) + if getattr(self.cfg, "xsa", 0): # r18: bez skladowej wzdluz wlasnego v tokenu (per glowa) + vn = F.normalize(v, dim=-1) + y = y - (y * vn).sum(-1, keepdim=True) * vn + y = y.transpose(1, 2).contiguous().view(B, T, D) + x = x + self.proj(y) + x = x + self.mlp(self.ln2(x)) + return x, v0 + + +TINYLM_SHA = "4876824de8c697a657e5f58351c216ffb84cb90d6a2a53ecd94933f7ddcc4cfa" # r19: Tiny-Chick-10M@60d32c4d tinylm_arch.py + + +def load_tinylm_module(path): + """r19: oryginalny tinylm_arch.py (Apache-2.0, kurcontko/Tiny-Chick-10M@60d32c4d) z pliku przypietego sha. + Bez --tinylm-file: tinylm_arch.py obok tego trenera (np. eval, ktory odtwarza model z configu ckpt).""" + import importlib.util + path = path or os.path.join(os.path.dirname(os.path.abspath(__file__)), "tinylm_arch.py") + if not os.path.exists(path): + raise SystemExit(f"STOP --arch tinylm: brak {path}") + with open(path, "rb") as f: + h = hashlib.sha256(f.read()).hexdigest() + if h != TINYLM_SHA: + raise SystemExit(f"STOP --arch tinylm: sha {path} = {h[:8]} != {TINYLM_SHA[:8]}") + spec = importlib.util.spec_from_file_location("tinylm_arch_tc", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +class GPT(nn.Module): + def __init__(self, vocab, n_layer, n_embd, n_head, block, cfg): + super().__init__() + self.cfg = cfg + if getattr(cfg, "arch", "gpt") == "tinylm": # r19: caly model = TinyLM z pliku Tiny Chick (klucze "tl.*") + tlm = load_tinylm_module(getattr(cfg, "tinylm_file", None)) + self.tl = tlm.TinyLM(tlm.TinyConfig( + vocab_size=vocab, d=n_embd, layers=n_layer, heads=n_head, + kv_heads=int(getattr(cfg, "kv_heads", 0) or 0) or n_head, mlp_mult=cfg.ffn_mult, act=cfg.ffn, + qk_norm=bool(cfg.qk_norm), qk_gain=1.0, rope_dims=int(getattr(cfg, "rope_dims", 0) or 0), + xsa=bool(getattr(cfg, "xsa", 0)), zero_init=bool(getattr(cfg, "zero_init", 0)), + max_position_embeddings=block, rope_theta=cfg.rope_theta, eps=cfg.norm_eps)) + self.block = block + return + E = int(getattr(cfg, "emb_dim", 0) or 0) # r15: faktoryzowane osadzenia (0 = jak r14) + self.emb_dim = E + self.tok = nn.Embedding(vocab, E if E else n_embd) + self.use_rope = cfg.pos == "rope" + if not self.use_rope: + self.pos = nn.Embedding(block, n_embd) + layout = getattr(cfg, "mixer_layout", None) or "A" * n_layer + if len(layout) != n_layer or set(layout) - {"A", "G"}: + raise SystemExit(f"STOP: --mixer-layout {layout!r}: dlugosc {n_layer}, tylko litery A/G") + nrot = int(getattr(cfg, "rope_dims", 0) or 0) + kvh = int(getattr(cfg, "kv_heads", 0) or 0) # r23: GQA dla blokow A + if kvh < 0 or kvh > n_head or (kvh and n_head % kvh): + raise SystemExit(f"STOP: --kv-heads {kvh}: 0 (MHA) albo dzielnik n_head {n_head}") + if nrot and (nrot % 2 or not 0 < nrot < n_embd // n_head or cfg.pos != "rope"): + raise SystemExit(f"STOP: --rope-dims {nrot}: parzyste, 0 < N < head_dim {n_embd // n_head}, tylko z --pos rope") + first_a = layout.find("A") # value-residual: v0 z pierwszego bloku A (r12: blok 0) + self.blocks = nn.ModuleList([Block(n_embd, n_head, block, cfg, is_first=(i == first_a), mixer=layout[i], layer_idx=i) + for i in range(n_layer)]) + self.lnf = make_norm(n_embd, cfg) + self.head = nn.Linear(E if E else n_embd, vocab, bias=False) + self.head.weight = self.tok.weight # tie + if E: + self.emb_up = nn.Linear(E, n_embd, bias=False) + self.emb_down = nn.Linear(n_embd, E, bias=False) + self.block = block + self.apply(self._init) + if getattr(cfg, "zero_init", 0): # r18: wyjscia galezi rezydualnych od zera (bloki G: proj fla bez zmian) + for b in self.blocks: + if b.mixer == "A": + nn.init.zeros_(b.proj.weight) + nn.init.zeros_(b.mlp.down.weight if hasattr(b.mlp, "down") else b.mlp[2].weight) + if E: + nn.init.normal_(self.emb_up.weight, 0.0, E ** -0.5) + nn.init.normal_(self.emb_down.weight, 0.0, n_embd ** -0.5) + + def _head(self, h): + return self.head(self.emb_down(h)) if self.emb_dim else self.head(h) + + def _init(self, m): + if getattr(m, "_r13_fla_init", False): # r13: moduly GatedDeltaNet2 zostaja z inicjalizacja fla + return + if isinstance(m, nn.Linear): + nn.init.normal_(m.weight, 0.0, 0.02) + if m.bias is not None: + nn.init.zeros_(m.bias) + elif isinstance(m, nn.Embedding): + nn.init.normal_(m.weight, 0.0, 0.02) + + def _loss_sum(self, h, y, t=None): + """r11: suma CE (+ z-loss) dla kawalka pozycji; h po lnf [B,t,D], y [B,t]. Ta sama arytmetyka co sciezka r10. + r14: t = logity nauczyciela tego kawalka [B,t,V] -> (suma straty mieszanej KD, suma samej CE).""" + logits = self._head(h) + cap = getattr(self.cfg, "logit_cap", 0.0) + if cap and cap > 0: + logits = cap * torch.tanh(logits / cap) + flat = logits.view(-1, logits.size(-1)) + s = F.cross_entropy(flat, y.reshape(-1), reduction="sum") + zc = getattr(self.cfg, "z_loss", 0.0) + if zc and zc > 0: + lse = torch.logsumexp(flat, dim=-1) + s = s + zc * (lse * lse).sum() + if t is None: + return s + return self._kd_mix(s, flat, t, 1), s + + def _kd_mix(self, ce, flat, t, div): + """r14: (1-A)*ce + A*T^2*KL(p_naucz || p_ucz); KL sumowane po pozycjach kawalka / div (1 = suma, N = srednia). + fp32, pelny vocab; flat = logity ucznia [N,V] (po logit_cap), t = logity nauczyciela [...,V].""" + A, T = self.cfg.kd_alpha, self.cfg.kd_temp + ls = F.log_softmax(flat.float() / T, dim=-1) + lt = F.log_softmax(t.reshape(-1, t.size(-1)).float() / T, dim=-1) + kl = (lt.exp() * (lt - ls)).sum() / div + return (1.0 - A) * ce + (A * T * T) * kl + + def forward(self, idx, targets=None, teacher_logits=None): + if getattr(self.cfg, "arch", "gpt") == "tinylm": # r19: ich forward (logity z tied emb), CE w fp32 jak u nich + logits = self.tl(idx).logits + if targets is None: + return logits, None + return logits, F.cross_entropy(logits.reshape(-1, logits.size(-1)).float(), targets.reshape(-1)) + B, T = idx.size() + x = self.tok(idx) + if self.emb_dim: + x = self.emb_up(x) + if not self.use_rope: + pos = torch.arange(T, device=idx.device) + x = x + self.pos(pos)[None] + v0 = None + act_ckpt = self.training and getattr(self.cfg, "act_ckpt", False) + for _ in range(getattr(self.cfg, "loop", 1)): + for b in self.blocks: + x, v0 = checkpoint(b, x, v0, use_reentrant=False) if act_ckpt else b(x, v0) + h = self.lnf(x) + n = getattr(self.cfg, "loss_chunk", 0) + if targets is not None and n and n < T: + if teacher_logits is None: + s = sum(checkpoint(self._loss_sum, h[:, i:i + n], targets[:, i:i + n], use_reentrant=False) + for i in range(0, T, n)) + return None, s / (B * T) + parts = [checkpoint(self._loss_sum, h[:, i:i + n], targets[:, i:i + n], teacher_logits[:, i:i + n], + use_reentrant=False) for i in range(0, T, n)] + return None, sum(p[0] for p in parts) / (B * T), sum(p[1] for p in parts).detach() / (B * T) + logits = self._head(h) + cap = getattr(self.cfg, "logit_cap", 0.0) + if cap and cap > 0: + logits = cap * torch.tanh(logits / cap) + loss = None + if targets is not None: + flat = logits.view(-1, logits.size(-1)) + loss = F.cross_entropy(flat, targets.view(-1)) + zc = getattr(self.cfg, "z_loss", 0.0) + if zc and zc > 0: + lse = torch.logsumexp(flat, dim=-1) + loss = loss + zc * (lse * lse).mean() + if teacher_logits is not None: + return logits, self._kd_mix(loss, flat, teacher_logits, flat.size(0)), loss.detach() + return logits, loss + + +def main(): + a = get_args() + world = int(os.environ.get("WORLD_SIZE", "1")) + rank = int(os.environ.get("RANK", "0")) + local_rank = int(os.environ.get("LOCAL_RANK", "0")) + ddp = world > 1 + if ddp: + if a.batch % world: + raise SystemExit(f"STOP: --batch {a.batch} (globalny) nie dzieli sie przez liczbe procesow {world}") + if torch.cuda.is_available(): + torch.cuda.set_device(local_rank) + dist.init_process_group("nccl" if torch.cuda.is_available() else "gloo") + if a.grad_accum < 1 or (a.batch // world) % a.grad_accum: + raise SystemExit(f"STOP: --grad-accum {a.grad_accum}: wiersze ranku {a.batch // world} nie dziela sie na K mikrobatchy") + master = rank == 0 + rows = slice(rank * (a.batch // world), (rank + 1) * (a.batch // world)) # wiersze globalnego batcha tego ranku + out_dir = a.out_dir or os.path.join(a.data_dir, "runs", a.run_id) + os.makedirs(out_dir, exist_ok=True) + log_path = os.path.join(out_dir, "train.log") + metrics_path = os.path.join(out_dir, "metrics.jsonl") + ckpt_path = os.path.join(out_dir, "ckpt.pt") + + def log(msg): + if not master: + return + line = f"[{time.strftime('%H:%M:%S')}] {msg}" + print(line, flush=True) + with open(log_path, "a", encoding="utf-8") as f: + f.write(line + "\n") + + events_path = a.events_jsonl + + def emit(kind, step, metrics=None, **extra): + if not events_path or not master: + return + rec = {"kind": kind, "updates": int(step), "tokens": int(step) * a.block * a.batch} + if metrics: + rec["metrics"] = metrics + rec.update(extra) + with open(events_path, "a", encoding="utf-8") as f: + f.write(json.dumps(rec) + "\n") + + torch.manual_seed(a.seed) + torch.backends.cuda.matmul.allow_tf32 = bool(a.tf32) # r17: --tf32 0 jak Qielbas M2 (domyslnie 1 = jak dotad) + torch.backends.cudnn.allow_tf32 = bool(a.tf32) + device = torch.device(f"cuda:{local_rank}" if torch.cuda.is_available() else "cpu") + use_bf16 = device.type == "cuda" and torch.cuda.is_bf16_supported() + ptdtype = torch.bfloat16 if use_bf16 else torch.float32 + ctx = torch.amp.autocast(device_type=device.type, dtype=ptdtype) if device.type == "cuda" else nullcontext() + log(f"device={device} bf16={use_bf16} dev={torch.cuda.get_device_name(local_rank) if device.type=='cuda' else 'cpu'}" + f" world={world}") + + if a.synthetic: + rng = np.random.default_rng(a.seed) + train_data = rng.integers(0, 256, size=8_000_000, dtype=np.uint8) + val_data = train_data[:200_000] + log("SYNTHETIC uint8 (smoke throughput, zero danych PII)") + else: + train_data = np.memmap(os.path.join(a.data_dir, "train.bin"), dtype=np.dtype(a.dtype), mode="r") + val_data = np.memmap(os.path.join(a.data_dir, "val.bin"), dtype=np.dtype(a.dtype), mode="r") + train2_data = (np.memmap(os.path.join(a.data_dir_2, "train.bin"), dtype=np.dtype(a.dtype), mode="r") + if a.data_dir_2 else None) + log(f"dane: train={len(train_data):,}B block={a.block} batch={a.batch} tok/step={a.block*a.batch:,}") + + class EpochSampler: + """Losowanie BEZ zwracania: epoka e = permutacja okien na siatce co `block` tokenow, przesunietej o losowe + offset_e w [0, block) (kolejne epoki widza inne granice okien). Krok s bierze okna s*batch..(s+1)*batch-1 + z ciaglego strumienia epok, wiec start od dowolnego kroku (resume) daje dokladnie ten sam ciag co run ciagly. + Epoka e ma wlasny RNG ziarna (seed, e): najpierw offset, potem permutacja; niezalezna od innych epok.""" + def __init__(self, n_tokens, block, batch, seed): + self.n, self.block, self.batch, self.seed = n_tokens, block, batch, seed + self._perm_e, self._perm = None, None + + def _meta(self, e): + rng = np.random.default_rng([self.seed, e]) + off = int(rng.integers(self.block)) + return rng, off, (self.n - 1 - off) // self.block # okno = block+1 tokenow od off + k*block + + def _perm_of(self, e): + if self._perm_e != e: # trzymamy tylko biezaca epoke (~70 MB przy 9.4B tok / 1024) + rng, _, nwin = self._meta(e) + self._perm_e, self._perm = e, rng.permutation(nwin) + return self._perm + + def starts(self, step): + g, need, out, e = step * self.batch, self.batch, [], 0 + while g >= self._meta(e)[2]: # epoki maja rozna dlugosc (offset), wiec liczymy po kolei + g -= self._meta(e)[2]; e += 1 + while need: + _, off, nwin = self._meta(e) + take = min(need, nwin - g) + out.append(off + self._perm_of(e)[g:g + take].astype(np.int64) * self.block) + need -= take; g = 0; e += 1 + return np.concatenate(out) + + class QueueSampler: + """r7: pula jako kolejka shardow (zakresy tokenow [lo,hi) w train.bin, wyrownane do dokumentow). + schedule = rotacje manifestu po kolei; slot j = kroki [j*S,(j+1)*S) na shardzie schedule[j]. W slocie kazde okno + uzyte najwyzej raz: siatka co `block` od lo+off, off i permutacja z RNG (seed, j, id shardu). Starty okien zaleza + tylko od (manifest, seed, krok), wiec resume z ckpt na granicy slotu jest dokladny, a usuniecie shardu ze slotu j + nie zmienia krokow < j*S. prefix_sha(krok) = hash slotow juz uzytych; zapisywany w ckpt i sprawdzany przy resume.""" + def __init__(self, manifest, n_tokens, block, batch, seed): + self.S, self.block, self.batch, self.seed = int(manifest["steps_per_shard"]), block, batch, seed + self.shards = manifest["shards"] + self.schedule = [sid for rot in manifest["rotations"] for sid in rot] + for sid in self.schedule: + sh = self.shards[sid] + if not 0 <= sh["lo"] < sh["hi"] <= n_tokens: + raise SystemExit(f"STOP kolejka: shard {sid} [{sh['lo']},{sh['hi']}) poza train.bin ({n_tokens})") + if (sh["hi"] - sh["lo"] - block) // block < self.S * batch: + raise SystemExit(f"STOP kolejka: shard {sid} ma za malo okien na {self.S} krokow x {batch}") + used = sorted((self.shards[s]["lo"], self.shards[s]["hi"], s) for s in dict.fromkeys(self.schedule)) + for (_, hi0, s0), (lo1, _, s1) in zip(used, used[1:]): + if lo1 < hi0: + raise SystemExit(f"STOP kolejka: shardy {s0} i {s1} nakladaja sie w train.bin (dane uzyte dwa razy)") + self._cache = (None, None) + + def _slot(self, j): + if self._cache[0] != j: + sid = self.schedule[j] + sh = self.shards[sid] + idn = int(hashlib.sha256(sid.encode()).hexdigest()[:8], 16) + rng = np.random.default_rng([self.seed, j, idn]) + off = int(rng.integers(self.block)) + nwin = (sh["hi"] - sh["lo"] - 1 - off) // self.block + perm = rng.permutation(nwin)[: self.S * self.batch].astype(np.int64) + self._cache = (j, (sh["lo"] + off, perm)) + return self._cache[1] + + def starts(self, step): + j, k = divmod(step, self.S) + if j >= len(self.schedule): + raise SystemExit(f"STOP kolejka: krok {step} poza harmonogramem ({len(self.schedule)} slotow x {self.S})") + base, perm = self._slot(j) + return base + perm[k * self.batch:(k + 1) * self.batch] * self.block + + def prefix_sha(self, step): + used = -(-step // self.S) # sloty, ktorych kroki juz sie zaczely + spec = [self.S, self.seed, self.block, self.batch, + [[sid, self.shards[sid]["lo"], self.shards[sid]["hi"], self.shards[sid]["sha256"]] for sid in self.schedule[:used]]] + return hashlib.sha256(json.dumps(spec).encode()).hexdigest() + + def _make_batch_cpu(split, generator=None, ix=None, shard=False): + """Wektoryzowane budowanie batcha: JEDEN numpy fancy-index zamiast + python-loop per-item. sliding_window_view daje strided-view (N-block, block+1) + BEZ kopiowania; windows[ix] materializuje tylko wybrane wiersze naraz. + Zwraca (x,y) long CPU (pinned jesli cuda). Rozklad batchy IDENTYCZNY jak + stary torch.stack-loop: x=data[i:i+block], y=data[i+1:i+1+block]. ix: gotowe starty okien (sampler epoch). + shard=True (trening pod DDP): starty losowane dla CALEGO globalnego batcha jak w 1-GPU, rank bierze swoje wiersze.""" + data = train_data if split == "train" else (train2_data if split == "train2" else val_data) + if ix is None: + ix = torch.randint(len(data) - a.block - 1, (a.batch,), generator=generator) + if shard: + ix = ix[rows] + # (N-block, block+1) view; jeden fancy-index kopiuje wybrane okna + windows = np.lib.stride_tricks.sliding_window_view(data, a.block + 1) + sel = windows[ix.numpy()] # (batch, block+1) materialized + x = torch.from_numpy(sel[:, :-1].astype(np.int64)) # astype -> contiguous copy + y = torch.from_numpy(sel[:, 1:].astype(np.int64)) + if device.type == "cuda": + x = x.pin_memory(); y = y.pin_memory() + return x, y + + def _to_device(x, y): + if device.type == "cuda": + return x.to(device, non_blocking=True), y.to(device, non_blocking=True) + return x.to(device), y.to(device) + + def get_batch(split, generator=None): + return _to_device(*_make_batch_cpu(split, generator)) + + class Prefetcher: + """Async double-buffer: 1 background-thread buduje NASTEPNY batch na CPU + (pinned) podczas gdy GPU liczy biezacy. queue depth=2. Konsument robi + .next() -> H2D-copy (non_blocking) w watku glownym. Watek uzywa wlasnego + torch.Generator (seeded), wiec ciag train-batchy jest deterministyczny i + NIEZALEZNY od timingu watku oraz od RNG val-loopa (dystrybucja bez zmian). + sampler (EpochSampler) zamiast generatora: starty okien z kroku, liczonego od start_step.""" + def __init__(self, split, generator, depth=2, sampler=None, start_step=0): + self.split = split + self.gen = generator + self.sampler, self.step = sampler, start_step + self.q = queue.Queue(maxsize=depth) + self._stop = threading.Event() + self.t = threading.Thread(target=self._worker, daemon=True) + self.t.start() + + def _worker(self): + while not self._stop.is_set(): + try: + if self.sampler is not None and sampler2 is not None and self.step >= a.switch_step: + item = _make_batch_cpu("train2", ix=torch.from_numpy(sampler2.starts(self.step - a.switch_step)), + shard=True) + self.step += 1 + elif self.sampler is not None: + item = _make_batch_cpu(self.split, ix=torch.from_numpy(self.sampler.starts(self.step)), shard=True) + self.step += 1 + else: + item = _make_batch_cpu(self.split, self.gen, shard=True) + except Exception as e: # przekaz blad do konsumenta + self.q.put(e) + return + while not self._stop.is_set(): + try: + self.q.put(item, timeout=0.5) + break + except queue.Full: + continue + + def next(self): + item = self.q.get() + if isinstance(item, Exception): + raise item + return _to_device(*item) + + def close(self): + self._stop.set() + # opróżnij kolejke zeby watek nie zawisl na put() + try: + self.q.get_nowait() + except queue.Empty: + pass + + if a.arch == "tinylm": # r19: tylko kombinacje, ktore TinyLM odtwarza; reszta = STOP zamiast cichego pominiecia + bad = [n for n, v in (("--loop > 1", a.loop != 1), ("--mixer-layout", a.mixer_layout), ("--emb-dim", a.emb_dim), + ("--value-residual", a.value_residual), ("--kd-teacher", a.kd_teacher), + ("--act-ckpt", a.act_ckpt), ("--loss-chunk", a.loss_chunk), ("--logit-cap", a.logit_cap), + ("--z-loss", a.z_loss), ("--pos != rope", a.pos != "rope"), + ("--rope-style != half", a.rope_style != "half"), ("--norm != rmsnorm", a.norm != "rmsnorm"), + ("--attn-bias != 0", a.attn_bias != 0), ("--ffn gelu", a.ffn == "gelu"), + ("--init-from", a.init_from)) if v] + if bad: + raise SystemExit(f"STOP --arch tinylm: niezgodne {bad}") + if a.n_embd % a.n_head or a.n_head % (a.kv_heads or a.n_head): + raise SystemExit(f"STOP --arch tinylm: n_embd {a.n_embd} / n_head {a.n_head} / kv_heads {a.kv_heads}") + elif a.tinylm_file: + raise SystemExit("STOP: --tinylm-file tylko z --arch tinylm") + elif a.kv_heads == a.n_head: # r23: K = n_head to zwykla MHA -> jak 0 (wagi, config ckpt i wyjscia = r22) + a.kv_heads = 0 + raw_model = GPT(a.vocab, a.n_layer, a.n_embd, a.n_head, a.block, a).to(device) + nparam = sum(p.numel() for p in raw_model.parameters()) + log(f"model GPT-ref: {nparam/1e6:.1f}M param (L{a.n_layer} d{a.n_embd} h{a.n_head}" + + (f" kv{a.kv_heads}" if a.arch == "gpt" and a.kv_heads else "") + ")") + if a.arch == "tinylm": + log(f"r19 arch=tinylm (Tiny-Chick-10M@60d32c4d tinylm_arch.py {TINYLM_SHA[:8]}): P={nparam} " + f"kv_heads={a.kv_heads or a.n_head} ffn={a.ffn} x{a.ffn_mult} qk_norm={a.qk_norm} rope_dims={a.rope_dims} " + f"xsa={a.xsa} zero_init={a.zero_init} eps={a.norm_eps} theta={a.rope_theta}") + n_train = nparam + if a.train_only: # r21: zamrozenie wszystkiego poza dopasowanymi nazwami (wzor ft_arc11.py) + if not a.init_from: + raise SystemExit("STOP --train-only: wymaga --init-from") + if a.arch == "tinylm" or a.loop != 1 or a.mixer_layout or a.kd_teacher or a.compile or ddp: + raise SystemExit("STOP --train-only: kombinacja nietestowana (tinylm / loop / mixer-layout / kd / compile / DDP)") + rx = re.compile(a.train_only) + for name, p in raw_model.named_parameters(): # tied head/tok wystepuje raz, jako tok.weight + p.requires_grad_(bool(rx.search(name))) + n_train = sum(p.numel() for p in raw_model.parameters() if p.requires_grad) + if n_train == 0: + raise SystemExit(f"STOP --train-only: {a.train_only!r} nie pasuje do zadnego parametru") + if raw_model.tok.weight.requires_grad: + raise SystemExit(f"STOP --train-only: {a.train_only!r} chwyta osadzenia / tied head") + log(f"r21 train-only {a.train_only!r}: uczone {n_train} z {nparam}") + opt = build_optimizer(a, raw_model) + log(f"optimizer={a.optimizer}" + (f" muon_lr={a.muon_lr} (mult={a.muon_lr/a.lr:.1f}x)" if a.optimizer == "muon" else "")) + if a.act_ckpt or a.loss_chunk: + log(f"r11 pamiec: act_ckpt={a.act_ckpt} loss_chunk={a.loss_chunk}") + if a.imha_t < 0: + raise SystemExit("STOP: --imha-t >= 0") + if a.imha_t: + bad = [n for n, v in (("--xsa", a.xsa), ("--mixer-layout", a.mixer_layout), ("--arch tinylm", a.arch == "tinylm")) if v] + if bad: + raise SystemExit(f"STOP --imha-t: kombinacja nietestowana {bad}") + log(f"r22 IMHA: T={a.imha_t} w {a.n_layer} blokach (jawna macierz uwagi, entropia + bramka); P={nparam}") + if a.loop < 1: + raise SystemExit("STOP: --loop >= 1") + if a.loop > 1: + log(f"r12 loop={a.loop}: {a.n_layer} wspoldzielonych blokow x {a.loop} = glebokosc {a.n_layer * a.loop}") + if a.mixer_layout or a.attn_bias != 1 or a.rope_style != "interleaved": + log(f"r13: mixer_layout={a.mixer_layout or 'A' * a.n_layer} gdn_head_dim={a.gdn_head_dim} " + f"attn_bias={a.attn_bias} rope_style={a.rope_style}") + if a.xsa or a.rope_dims or a.zero_init or a.ffn == "relu2": + log(f"r18: xsa={a.xsa} rope_dims={a.rope_dims or 'wszystkie'} zero_init={a.zero_init} ffn={a.ffn} ffn_mult={a.ffn_mult}") + # DDP: identyczne wagi startowe na kazdym ranku (ten sam seed) + broadcast z rank 0 w konstruktorze DDP. + # broadcast_buffers=False: forward walidacji (bez backward) nie robi kolektywow. + model = DDP(raw_model, device_ids=[local_rank] if device.type == "cuda" else None, + broadcast_buffers=False) if ddp else raw_model + model = torch.compile(model) if a.compile else model + if a.compile: + log("torch.compile enabled") + + start_step = 0 + if a.resume and os.path.exists(ckpt_path): + ck = torch.load(ckpt_path, map_location=device, weights_only=True) # r20 + raw_model.load_state_dict(ck["model"]); opt.load_state_dict(ck["opt"]); start_step = ck["step"] + if ck.get("config", {}).get("data_seed") != a.data_seed: # r23: ta sama kolejność danych po wznowieniu + raise SystemExit(f"STOP --resume: data_seed w ckpt {ck.get('config', {}).get('data_seed')} != {a.data_seed}") + log(f"RESUME @ {start_step}") + if a.init_from and start_step == 0: + ck0 = torch.load(a.init_from, map_location=device, weights_only=True) # r20 (r18/r19: False) + c0 = ck0.get("config", {}) + for key in ("vocab", "n_layer", "n_embd", "n_head"): + if c0.get(key) is not None and c0[key] != getattr(a, key): + raise SystemExit(f"STOP --init-from: {key}={c0[key]} w ckpt != {getattr(a, key)} w argumentach") + if c0.get("loop", 1) != a.loop: + raise SystemExit(f"STOP --init-from: loop={c0.get('loop', 1)} w ckpt != {a.loop} w argumentach") + for key, dflt in (("xsa", 0), ("rope_dims", 0), ("ffn", "gelu"), ("imha_t", 0), ("kv_heads", 0)): # r18: klucze bez wag tez musza sie zgadzac; r22: imha_t; r23: kv_heads + if c0.get(key, dflt) != getattr(a, key): + raise SystemExit(f"STOP --init-from: {key}={c0.get(key, dflt)} w ckpt != {getattr(a, key)} w argumentach") + raw_model.load_state_dict(ck0["model"], strict=True) + log(f"INIT-FROM {a.init_from} (wagi, krok 0, swiezy optymalizator; grown_from={ck0.get('grown_from')})") + del ck0 + teacher = None + if a.kd_teacher: # r14: nauczyciel KD (po budowie/wznowieniu ucznia, wiec jego wagi startowe bez zmian) + if not 0.0 <= a.kd_alpha <= 1.0 or a.kd_temp <= 0: + raise SystemExit(f"STOP --kd: alpha {a.kd_alpha} poza [0,1] albo temp {a.kd_temp} <= 0") + _h = hashlib.sha256() + with open(a.kd_teacher, "rb") as _f: + for _chunk in iter(lambda: _f.read(1 << 20), b""): + _h.update(_chunk) + _tsha = _h.hexdigest() + if _tsha != (a.kd_teacher_sha or "").lower(): + raise SystemExit(f"STOP --kd-teacher: sha {_tsha} != --kd-teacher-sha") + ckt = torch.load(a.kd_teacher, map_location="cpu", weights_only=True) + ct = ckt["config"] + if ct["vocab"] != a.vocab or ct["block"] < a.block: + raise SystemExit(f"STOP --kd-teacher: vocab {ct['vocab']} / block {ct['block']} nauczyciela " + f"vs vocab {a.vocab} / block {a.block} ucznia") + import types + with torch.random.fork_rng(devices=[]): # budowa (init) nauczyciela nie zmienia globalnego RNG ucznia + teacher = GPT(ct["vocab"], ct["n_layer"], ct["n_embd"], ct["n_head"], ct["block"], types.SimpleNamespace(**ct)) + teacher.load_state_dict({k.removeprefix("_orig_mod."): v for k, v in ckt["model"].items()}, strict=True) + teacher = teacher.to(device).eval().requires_grad_(False) + log(f"r14 KD: nauczyciel {a.kd_teacher} sha {_tsha[:8]} krok {ckt.get('step')} " + f"{sum(p.numel() for p in teacher.parameters())/1e6:.1f}M (L{ct['n_layer']} d{ct['n_embd']} " + f"{ct.get('mixer_layout') or 'A' * ct['n_layer']}), alpha {a.kd_alpha} T {a.kd_temp}") + del ckt + teacher_fwd = torch.compile(teacher) if a.compile else teacher + elif a.kd_alpha or a.kd_teacher_sha: + raise SystemExit("STOP: --kd-alpha/--kd-teacher-sha bez --kd-teacher") + if a.data_offset_steps and a.sampler != "epoch": + raise SystemExit("STOP: --data-offset-steps wymaga --sampler epoch") + + def _lr_sched(s): + if s < a.warmup: + return a.lr * (s + 1) / a.warmup + if s >= a.steps: + return a.min_lr + if a.schedule == "wsd": + # warmup-stable-decay (JugnuLM/MiniCPM): constant peak, then 1-sqrt decay in last decay-frac. + decay_start = int(a.steps * (1.0 - a.decay_frac)) + if s < decay_start: + return a.lr + d = (s - decay_start) / max(1, a.steps - decay_start) # 0->1 over decay phase + return a.min_lr + (a.lr - a.min_lr) * (1.0 - d ** 0.5) # 1-sqrt: fast early decay + r = (s - a.warmup) / max(1, a.steps - a.warmup) + return a.min_lr + 0.5 * (a.lr - a.min_lr) * (1 + math.cos(math.pi * r)) + + # --rewarm-steps: galaz z ckpt (np. cosine @320k -> wsd) nie skacze od razu na nowy harmonogram, + # tylko przechodzi liniowo od LR zapisanego w ckpt (param_groups po opt.load_state_dict). + rewarm_from = None + if start_step and a.rewarm_steps > 0: + _g0 = opt.param_groups[0] + rewarm_from = _g0["lr"] / _g0.get("lr_mult", 1.0) + log(f"rewarm: LR {rewarm_from:.3e} -> harmonogram przez {a.rewarm_steps} krokow od {start_step}") + + def lr_at(s): + base = _lr_sched(s) + if rewarm_from is not None and s < start_step + a.rewarm_steps: + f = (s - start_step + 1) / a.rewarm_steps + return rewarm_from + (base - rewarm_from) * f + return base + + @torch.no_grad() + def eval_val(): + model.eval() + ls = [] + for _ in range(a.eval_iters): + xb, yb = get_batch("val") + with ctx: + _, loss = model(xb, yb) + ls.append(loss.item()) + model.train() + return sum(ls) / len(ls) + + model.train() + log(f"START gpt-ref: steps={a.steps} (od {start_step}) lr={a.lr}->{a.min_lr}") + # dedykowany seeded generator dla train-prefetchera (determinizm niezalezny + # od RNG val-loopa i timingu watku; ta sama dystrybucja co global-RNG) + train_gen = torch.Generator() + ds = a.seed if a.data_seed is None else a.data_seed # r23: kolejność danych osobno od inicjalizacji + train_gen.manual_seed(ds) + sampler = None + sampler2 = None + if a.queue: + if a.synthetic: + raise SystemExit("STOP: --queue wymaga prawdziwego train.bin (bez --synthetic)") + with open(a.queue, "rb") as f: + _qraw = f.read() + sampler = QueueSampler(json.loads(_qraw), len(train_data), a.block, a.batch, ds) + if start_step: + _ckq = ck.get("queue") or {} + if _ckq.get("prefix_sha") != sampler.prefix_sha(start_step): + raise SystemExit(f"STOP kolejka: sloty uzyte do kroku {start_step} w manifescie != zapisane w ckpt " + "(resume nie bylby dokladny)") + log(f"sampler queue: manifest {hashlib.sha256(_qraw).hexdigest()[:8]}, {len(sampler.schedule)} slotow x " + f"{sampler.S} krokow, start od kroku {start_step} = slot {start_step // sampler.S}") + elif a.sampler == "epoch" and not a.synthetic: + sampler = EpochSampler(len(train_data), a.block, a.batch, ds) + _, _off0, _nwin0 = sampler._meta(0) + log(f"sampler epoch: {_nwin0:,} okien/epoke (offset0 {_off0}), start od kroku {start_step} " + f"= epoka {start_step * a.batch / _nwin0:.3f}") + if a.data_dir_2: + if a.switch_step <= 0 or _nwin0 // a.batch < a.switch_step: + raise SystemExit(f"STOP --data-dir-2: --switch-step {a.switch_step} poza epoka 0 train.bin " + f"({_nwin0 // a.batch} krokow)") + sampler2 = EpochSampler(len(train2_data), a.block, a.batch, ds) + _nwin2 = sampler2._meta(0)[2] + log(f"data-dir-2: {a.data_dir_2} {len(train2_data):,} tok, {_nwin2:,} okien/epoke = {_nwin2 // a.batch} krokow, " + f"od kroku strumienia {a.switch_step}") + elif a.data_dir_2: + raise SystemExit("STOP: --data-dir-2 wymaga --sampler epoch") + elif start_step and not a.synthetic: + # --resume: przewin generator o start_step batchy, zeby wznowiony run losowal DOKLADNIE te okna, + # ktore run ciagly losowalby od kroku start_step (bez tego resume powtarzal okna z krokow 0..N). + # Kazdy krok zuzywa 1 x torch.randint(len-block-1, (batch,)); 320k krokow ~1 s CPU. + _n = len(train_data) - a.block - 1 + for _ in range(start_step): + torch.randint(_n, (a.batch,), generator=train_gen) + log(f"train RNG fast-forward o {start_step} krokow (resume)") + if a.data_offset_steps: + log(f"data-offset: dane od kroku strumienia {start_step + a.data_offset_steps} (krok biegu {start_step} + {a.data_offset_steps})") + prefetcher = Prefetcher("train", train_gen, sampler=sampler, start_step=start_step + a.data_offset_steps) + end_step = a.steps + if a.queue and a.queue_one_slot: + end_step = min(a.steps, (start_step // sampler.S + 1) * sampler.S) + if sampler2 is not None: + _last = a.data_offset_steps + end_step - 1 # ostatni krok strumienia tego biegu + if _last - a.switch_step >= sampler2._meta(0)[2] // a.batch: + raise SystemExit(f"STOP --data-dir-2: krok strumienia {_last} wychodzi poza 1 epoke DATA_DIR_2 (powtorzenie)") + + if a.batch_sha: + _want = sorted({int(s) for s in a.batch_sha.split(",")}) + if _want[0] < start_step or _want[-1] >= end_step: + raise SystemExit(f"STOP --batch-sha: kroki poza [{start_step}, {end_step})") + for s in range(start_step, _want[-1] + 1): + xb, yb = prefetcher.next() + if s in _want: + _h = hashlib.sha256(xb.cpu().numpy().tobytes() + yb.cpu().numpy().tobytes()).hexdigest() + log(f"BATCH-SHA krok biegu {s} = krok strumienia {s + a.data_offset_steps}: {_h}") + prefetcher.close() + log("BATCH-SHA DONE") + return + + def save_ckpt(done): + if master: + state = {"model": raw_model.state_dict(), "opt": opt.state_dict(), "step": done, + "config": {"vocab": a.vocab, "n_layer": a.n_layer, "n_embd": a.n_embd, + "n_head": a.n_head, "block": a.block, "norm": a.norm, + "norm_eps": a.norm_eps, "pos": a.pos, "rope_theta": a.rope_theta, + "ffn": a.ffn, "ffn_mult": a.ffn_mult, "value_residual": a.value_residual, + "qk_norm": a.qk_norm, "logit_cap": a.logit_cap, "z_loss": a.z_loss, + **({"loop": a.loop} if a.loop > 1 else {}), + **({"data_seed": a.data_seed} if a.data_seed is not None else {}), + **({"mixer_layout": a.mixer_layout, "gdn_head_dim": a.gdn_head_dim} if a.mixer_layout else {}), + **({"attn_bias": a.attn_bias} if a.attn_bias != 1 else {}), + **({"rope_style": a.rope_style} if a.rope_style != "interleaved" else {}), + **({"emb_dim": a.emb_dim} if a.emb_dim else {}), + **({"xsa": a.xsa} if a.xsa else {}), + **({"rope_dims": a.rope_dims} if a.rope_dims else {}), + **({"zero_init": a.zero_init} if a.zero_init else {}), + **({"imha_t": a.imha_t} if a.imha_t else {}), + **({"kv_heads": a.kv_heads} if a.arch == "gpt" and a.kv_heads else {}), + **({"arch": "tinylm", "kv_heads": a.kv_heads or a.n_head, "tinylm_sha256": TINYLM_SHA} + if a.arch == "tinylm" else {})}} + if a.queue: + state["queue"] = {"prefix_sha": sampler.prefix_sha(done), "slots_used": -(-done // sampler.S)} + if teacher is not None: + state["kd"] = {"teacher": a.kd_teacher, "teacher_sha256": _tsha, "alpha": a.kd_alpha, "temp": a.kd_temp} + if a.train_only: # r21: poza "config", wiec --init-from i valport bez zmian + state["train_only"] = {"regex": a.train_only, "n_trainable": n_train} + torch.save(state, ckpt_path + ".tmp") + os.replace(ckpt_path + ".tmp", ckpt_path) # crash w trakcie zapisu nie zostawia ucietego ckpt.pt + log(f"ckpt @ {done}") + if a.events_jsonl: + _h = hashlib.sha256() + with open(ckpt_path, "rb") as _cf: + for _chunk in iter(lambda: _cf.read(1 << 20), b""): + _h.update(_chunk) + emit("checkpoint", done, sha256=_h.hexdigest()) + if ddp: + dist.barrier() # inne ranki nie ida dalej (ani nie koncza), zanim rank 0 zapisze ckpt + + def finish(): + prefetcher.close() + if ddp: + dist.destroy_process_group() + + # r9 (Monter, 06.10): strata sumowana na urządzeniu (float64), .item() tylko przy logu. r8 robił loss.item() co krok, + # czyli synchronizację GPU->CPU co krok, przez co CPU nie mógł kolejkować następnego kroku. Obliczenia modelu, + # kolejność batchy i aktualizacje wag są identyczne z r8; zmienia się tylko moment odczytu straty do logu. + t0 = time.time(); running = torch.zeros((), dtype=torch.float64, device=device) + running_ce = torch.zeros((), dtype=torch.float64, device=device) + for step in range(start_step, end_step): + lr = lr_at(step) + for g in opt.param_groups: + g["lr"] = lr * g.get("lr_mult", 1.0) + xb, yb = prefetcher.next() + if a.grad_accum == 1: + with ctx: + if teacher is None: + _, loss = model(xb, yb) + else: + with torch.no_grad(): + tl = teacher_fwd(xb)[0] + _, loss, ce = model(xb, yb, tl) + del tl + running_ce += ce.to(torch.float64) + loss.backward() + else: # r16: K mikrobatchy z wierszy ranku, gradient = srednia (strata_j / K); DDP synchronizuje tylko ostatni + K = a.grad_accum; m = xb.size(0) // K + loss = torch.zeros((), dtype=torch.float32, device=device) + for j in range(K): + xj, yj = xb[j * m:(j + 1) * m], yb[j * m:(j + 1) * m] + with (model.no_sync() if ddp and j < K - 1 else nullcontext()): + with ctx: + if teacher is None: + _, lj = model(xj, yj) + else: + with torch.no_grad(): + tl = teacher_fwd(xj)[0] + _, lj, ce = model(xj, yj, tl) + del tl + running_ce += ce.to(torch.float64) / K + (lj / K).backward() + loss += lj.detach().float() / K + gn = torch.nn.utils.clip_grad_norm_(model.parameters(), a.grad_clip) if a.grad_clip > 0 else 0.0 + opt.step(); opt.zero_grad(set_to_none=True) + running += loss.detach().to(torch.float64) + if (step + 1) % a.log_every == 0: + if ddp: # strata logowana = srednia po wszystkich wierszach globalnego batcha (jak w 1-GPU) + dist.all_reduce(running) + running = running / world + if teacher is not None: + dist.all_reduce(running_ce) + running_ce = running_ce / world + running = running.item() + kd_ce = running_ce.item() / a.log_every if teacher is not None else None + dt = time.time() - t0 + tok_s = a.log_every * a.block * a.batch / dt + mem = torch.cuda.max_memory_allocated(device)/1e9 if device.type == "cuda" else 0.0 + log(f"step {step+1}/{a.steps} loss {running/a.log_every:.4f} lr {lr:.2e} gnorm {float(gn):.2f} {tok_s:,.0f} tok/s peakVRAM {mem:.1f}GB" + + (f" ce {kd_ce:.4f}" if kd_ce is not None else "")) + if master: + with open(metrics_path, "a", encoding="utf-8") as f: + f.write(json.dumps({"step": step+1, "loss": running/a.log_every, "lr": lr, "tok_s": tok_s, + **({"ce": kd_ce} if kd_ce is not None else {})}) + "\n") + emit("update", step + 1, metrics={"loss": running / a.log_every, "tokens_per_second": tok_s, + "gradient_norm": float(gn), "learning_rate": lr}) + running = torch.zeros((), dtype=torch.float64, device=device); t0 = time.time() + running_ce = torch.zeros((), dtype=torch.float64, device=device) + if (step + 1) % a.eval_every == 0: + vloss = eval_val() + log(f" >> VAL loss {vloss:.4f} @ {step+1}") + emit("evaluation", step + 1, metrics={"loss": vloss}) + if (step + 1) % a.ckpt_every == 0 and not a.synthetic: + save_ckpt(step + 1) + if end_step % a.ckpt_every and (a.queue or not a.synthetic): + save_ckpt(end_step) # r20: zawsze ckpt po ostatnim kroku poza rytmem ckpt_every (r18/r19: tylko z --queue; k9 13.10: 1140 zamiast 1145) + if end_step < a.steps: + log(f"SLOT DONE @ {end_step}") + finish() + return + log("DONE") + emit("end", a.steps, status="completed") + finish() + + +if __name__ == "__main__": + main() diff --git a/kod/uruchom_ramie_v03.sh b/kod/uruchom_ramie_v03.sh new file mode 100644 index 0000000000000000000000000000000000000000..b98f34e6b4f35084948fb0003b21e10d1a89bfdb --- /dev/null +++ b/kod/uruchom_ramie_v03.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# uruchom_ramie_v03.sh (Monter, 22.10): jak uruchom_ramie.sh (95eb583b), ale ramie_v03.sh (v0.1 + CKPT / RESUME, tylko I/O) +# i kopiarka historii ckpt (kopiarka_ckpt.sh) na macierz /var/lib/docker/kolektyw/dowody400m_ckpt/. +# Osobny plik, bo działające ramiona fali 1 czytają uruchom_ramie.sh w trakcie (bash czyta skrypt przyrostowo). +# użycie: [CKPT=500] [RESUME=0|1] [MINI_STEPS=20] [MB=16] bash uruchom_ramie_v03.sh [MINI=0|1] +set -euo pipefail +ARM=$1; GPU=$2; MINI=${3:-0} +K="$HOME/kolektyw/dowody400m/kod" +export D="$HOME/kolektyw/dane_dowody/out_v1_fw2" +export PY="$HOME/kolektyw/polocr/venv/bin/python" T="$K/train_gpt_ref_r23.py" V="$K/val_bpb.py" TOK="$D/tokenizer.json" +export O="${O:-$HOME/kolektyw/dowody400m/wyniki}" ARM GPU MINI DATA_SEED=20261022 +sumy() { awk -v f="$1" '$2==f || $2=="*"f {print $1}' "$D/SHA256SUMS"; } +export TRAIN_SHA=$(sumy train.bin) VALPL_SHA=$(sumy val_pl.bin) VALEN_SHA=$(sumy val_en.bin) +for s in "$TRAIN_SHA" "$VALPL_SHA" "$VALEN_SHA"; do [[ $s =~ ^[0-9a-f]{64}$ ]] || { echo "STOP: brak sumy w $D/SHA256SUMS"; exit 1; }; done +R=$HOME/kolektyw/polocr/pydev/root/usr/include +[ "$(sha256sum "$K/ramie_v03.sh" | cut -c1-8)" = "${RAMIE_V03_SHA:?RAMIE_V03_SHA = 8 znaków sha ramie_v03.sh}" ] \ + || { echo "STOP: ramie_v03.sh != $RAMIE_V03_SHA"; exit 1; } +export CPATH=$R/python3.12:$R/x86_64-linux-gnu/python3.12:$R +OUT="$O/$ARM"; [ "$MINI" = 1 ] && OUT="$O/mini_$ARM" +DEST="/var/lib/docker/kolektyw/dowody400m_ckpt/$(basename "$O")_$(basename "$OUT")" +mkdir -p "$O" "$OUT" +ENV="$O/srodowisko_${ARM}_mini${MINI}_resume${RESUME:-0}.txt" +{ echo "data $(date -u +%FT%TZ) ARM $ARM GPU $GPU MINI $MINI DATA_SEED $DATA_SEED CKPT ${CKPT:-500} RESUME ${RESUME:-0} DEST $DEST" + echo "CPATH=$CPATH" + sha256sum "$K/ramie_v03.sh" "$K/train_gpt_ref_r23.py" "$K/val_bpb.py" "$K/kopiarka_ckpt.sh" "$D/SHA256SUMS" + "$PY" -c "import torch; print(torch.__version__, torch.version.cuda)" + "$PY" -m pip freeze + nvidia-smi --query-gpu=index,name,memory.used,driver_version --format=csv,noheader; } > "$ENV" 2>&1 \ + || { echo "STOP: zapis srodowiska nieudany ($ENV)"; exit 1; } +rm -f "$OUT/.kopiarka_stop" +bash "$K/kopiarka_ckpt.sh" "$OUT" "$DEST" & +KP=$! +rc=0; bash "$K/ramie_v03.sh" || rc=$? +touch "$OUT/.kopiarka_stop"; wait "$KP" || true +cp "$ENV" "$OUT/srodowisko_resume${RESUME:-0}.txt" +exit $rc diff --git a/kod/val_bpb.py b/kod/val_bpb.py new file mode 100644 index 0000000000000000000000000000000000000000..b08eda3ddaa2aec6c2f33526e8c0ecf05daf90b5 --- /dev/null +++ b/kod/val_bpb.py @@ -0,0 +1,149 @@ +"""val_bpb.py v0 (Hart, N-02, 2026-10-20) - strata na bajt (bits per byte) na val dla ramion dnia dowodów 400M +(karta Mierniczego Mierniczy/miary/karta_dzien_dowodow_400m_v0.md v0.1 §2; warunek 3 z 20:56Z: suma log-prawdopodobieństw +podzielona przez bajty UTF-8 tekstu, nie przez tokeny). + +Ten sam sposób liczenia co val500m.py v1 (f3724b9b, bez zmian, bo jego sha jest przypięty w start_500m.sh): okna +niezachodzące od offsetu 0 przez cały val.bin, x = tok[o:o+W], y = tok[o+1:o+W+1]; fp32, tf32 wyłączone, bez autocast; +model z ckpt['config'] przez klasę GPT trenera z pinów (r20, r21, r23; r23 obsługuje kv_heads), strict load. +Do tego: bajty UTF-8 każdego tokenu celu z tokenizer.json (BPE bajtowy jak v6: odwrócona mapa ByteLevel GPT-2; tokeny +specjalne = 0 bajtów) i wynik + bpb = Σ CE(nat) / ln 2 / Σ bajtów celów (wszystkie cele okien, także EOS; jednakowo dla wszystkich ramion) +oraz średnia CE na token. JSON 'val_bpb_v0': ckpt / val / tokenizer / skrypt / trener sha256, okno, liczba okien, sumy +nat i bajtów, bpb, ce_token. Istniejący --out = STOP. + python val_bpb.py --ckpt C --ckpt-sha S --trainer train_gpt_ref_r23.py --val val_pl.bin --val-sha S + --tokenizer tok_32768.json --tokenizer-sha S --dtype uint16 --out val_pl_.json [--bs 4] +""" +import argparse +import hashlib +import importlib.util +import json +import math +import os +import sys +from types import SimpleNamespace + +import numpy as np +import torch +import torch.nn.functional as F + +TRAINERS = {"398fd21eb0f567a20c96b535001fd89dbbd40b0df502a70af20e410973c122a3": "r20", + "ca9ee23caba1418c8016a7075cd1f1cf1d276babc41d3b9a703d4c30b526e9de": "r21", + "db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b": "r23"} + + +def die(msg): + print(f"STOP: {msg}") + sys.exit(1) + + +def sha_file(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for b in iter(lambda: f.read(1 << 24), b""): + h.update(b) + return h.hexdigest() + + +def bajty_tokenow(tok_json, vocab): + """Długość w bajtach UTF-8 każdego id z tokenizer.json (BPE ByteLevel); specjalne = 0.""" + t = json.load(open(tok_json, encoding="utf-8")) + if (t.get("pre_tokenizer") or {}).get("type") not in ("ByteLevel", "Sequence") or t["model"]["type"] != "BPE": + die("tokenizer nie jest BPE z ByteLevel") + bs = list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1)) + cs, n = bs[:], 0 + for b in range(256): + if b not in bs: + bs.append(b) + cs.append(256 + n) + n += 1 + znak_na_bajt = {chr(c): b for b, c in zip(bs, cs)} + dl = np.zeros(vocab, dtype=np.int64) + for s, i in t["model"]["vocab"].items(): + if i < vocab: + dl[i] = len(s) if all(ch in znak_na_bajt for ch in s) else 0 + for at in t.get("added_tokens", []): + if at["id"] < vocab: + dl[at["id"]] = 0 + return dl + + +def main(): + p = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + p.add_argument("--ckpt", required=True) + p.add_argument("--ckpt-sha", required=True) + p.add_argument("--trainer", required=True) + p.add_argument("--val", required=True) + p.add_argument("--val-sha", required=True) + p.add_argument("--tokenizer", required=True) + p.add_argument("--tokenizer-sha", required=True) + p.add_argument("--dtype", required=True, choices=("uint16", "uint32")) + p.add_argument("--out", required=True) + p.add_argument("--device", default="cuda") + p.add_argument("--bs", type=int, default=4) + p.add_argument("--allow-cpu", action="store_true", help="tylko w suchym biegu / testach") + a = p.parse_args() + if a.device == "cpu" and not a.allow_cpu: + die("liczy na GPU; --allow-cpu tylko w testach") + if a.bs < 1: + die("--bs >= 1") + if os.path.exists(a.out): + die(f"{a.out} istnieje") + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + tr_sha = sha_file(a.trainer) + if tr_sha not in TRAINERS: + die(f"trener sha {tr_sha[:8]} poza pinami r20 / r21 / r23") + for nazwa, plik, oczek in (("val", a.val, a.val_sha), ("ckpt", a.ckpt, a.ckpt_sha), ("tokenizer", a.tokenizer, a.tokenizer_sha)): + if sha_file(plik) != oczek.lower(): + die(f"{nazwa} sha z bajtów != podanego") + spec = importlib.util.spec_from_file_location("trainer_pin", a.trainer) + T = importlib.util.module_from_spec(spec) + spec.loader.exec_module(T) + ck = torch.load(a.ckpt, map_location="cpu", weights_only=True) + step, cfg = ck.get("step"), SimpleNamespace(**ck["config"]) + if getattr(cfg, "arch", "gpt") != "gpt": + die(f"arch {cfg.arch}: tylko arch=gpt") + dev = torch.device(a.device) + model = T.GPT(cfg.vocab, cfg.n_layer, cfg.n_embd, cfg.n_head, cfg.block, cfg).to(dev) + model.load_state_dict(ck["model"]) + del ck + model.float().eval() + P = sum(prm.numel() for prm in {id(x): x for x in model.parameters()}.values()) + dl = torch.from_numpy(bajty_tokenow(a.tokenizer, cfg.vocab)).to(dev) + W = cfg.block + data = np.memmap(a.val, dtype=np.dtype(a.dtype), mode="r") + offs = np.arange(0, max(0, len(data) - W), W, dtype=np.int64) + offs = offs[offs + W + 1 <= len(data)] + if len(offs) == 0: + die("val krótszy niż jedno okno") + if int(data.max()) >= cfg.vocab: + die(f"max id val {int(data.max())} >= vocab {cfg.vocab}") + nat, bajty, tokeny = 0.0, 0, 0 + with torch.no_grad(): + for i in range(0, len(offs), a.bs): + chunk = offs[i:i + a.bs] + x = torch.from_numpy(np.stack([np.asarray(data[o:o + W], dtype=np.int64) for o in chunk])).to(dev) + y = torch.from_numpy(np.stack([np.asarray(data[o + 1:o + W + 1], dtype=np.int64) for o in chunk])).to(dev) + logits, _ = model(x) + ce = F.cross_entropy(logits.float().reshape(-1, logits.size(-1)), y.reshape(-1), reduction="sum") + nat += float(ce) + bajty += int(dl[y].sum()) + tokeny += y.numel() + del logits, ce + if not math.isfinite(nat) or bajty == 0: + die("nieskończona strata albo 0 bajtów") + rep = {"schema": "val_bpb_v0", "ckpt_sha256": a.ckpt_sha.lower(), "step": step, "val_sha256": a.val_sha.lower(), + "tokenizer_sha256": a.tokenizer_sha.lower(), "script_sha256": sha_file(os.path.abspath(__file__)), + "trainer_sha256": tr_sha, "window": W, "n_windows": len(offs), "tokens": tokeny, "bytes": bajty, + "sum_nat": nat, "bpb": nat / math.log(2) / bajty, "ce_token": nat / tokeny, "P": P, + "dtype": "fp32, tf32 off, bez autocast", "torch": torch.__version__, + "device": torch.cuda.get_device_name(dev) if dev.type == "cuda" else "cpu"} + tmp = a.out + ".tmp" + with open(tmp, "w", encoding="utf-8") as f: + json.dump(rep, f) + os.replace(tmp, a.out) + print(f"VAL-BPB-OK step {step} bpb {rep['bpb']:.5f} ce {rep['ce_token']:.5f} bajty {bajty} P {P} -> {a.out}") + + +if __name__ == "__main__": + main() diff --git a/test_wznowienia/A/run/SHA256SUMS_kopiarka b/test_wznowienia/A/run/SHA256SUMS_kopiarka new file mode 100644 index 0000000000000000000000000000000000000000..d482565eceb1c9ce18e4ef6449703917b148d820 --- /dev/null +++ b/test_wznowienia/A/run/SHA256SUMS_kopiarka @@ -0,0 +1,2 @@ +9bc8d9445b7c2df14b8a4bf34a90e35095d8e7b31c8b27f18b878b563a87a75e ckpt_20.pt +f5877f84b95282b73d3b112299f14cd79aa28ca7a18d070ef91ee872eefb7d9c ckpt_40.pt diff --git a/test_wznowienia/A/run/args.txt b/test_wznowienia/A/run/args.txt new file mode 100644 index 0000000000000000000000000000000000000000..38f36de4f2c6b13a78a300bf2bb8b3dfc90faa66 --- /dev/null +++ b/test_wznowienia/A/run/args.txt @@ -0,0 +1 @@ +--data-dir /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2 --out-dir /home/ubuntu/kolektyw/dowody400m/test_resume/A/mini_R1 --run-id dowody_R1 --vocab 32768 --dtype uint16 --block 4096 --n-layer 16 --n-embd 896 --n-head 14 --value-residual --batch 128 --grad-accum 8 --norm rmsnorm --pos rope --rope-theta 500000 --ffn swiglu --ffn-mult 2.667 --qk-norm --attn-bias 0 --optimizer muon --muon-lr 0.02 --lr 6e-4 --min-lr 6e-05 --wd 0.1 --schedule wsd --decay-frac 0.2 --warmup 500 --act-ckpt --loss-chunk 1024 --sampler random --seed 1 --data-seed 20261022 --compile --log-every 20 --eval-every 41 --eval-iters 1 --steps 40 --ckpt-every 20 \ No newline at end of file diff --git a/test_wznowienia/A/run/metrics.jsonl b/test_wznowienia/A/run/metrics.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..eb815551dbb448a2a6fab176f0863111a53f52f2 --- /dev/null +++ b/test_wznowienia/A/run/metrics.jsonl @@ -0,0 +1,2 @@ +{"step": 20, "loss": 10.375722694396973, "lr": 2.3999999999999997e-05, "tok_s": 44100.237652083095} +{"step": 40, "loss": 9.556375885009766, "lr": 4.7999999999999994e-05, "tok_s": 53978.165094816744} diff --git a/test_wznowienia/A/run/srodowisko_R1_mini1_resume0.txt b/test_wznowienia/A/run/srodowisko_R1_mini1_resume0.txt new file mode 100644 index 0000000000000000000000000000000000000000..b9512da35b06a2236cf8342bbaf93278ff14b9ff --- /dev/null +++ b/test_wznowienia/A/run/srodowisko_R1_mini1_resume0.txt @@ -0,0 +1,64 @@ +data 2026-10-23T02:33:05Z ARM R1 GPU 0 MINI 1 DATA_SEED 20261022 CKPT 20 RESUME 0 DEST /var/lib/docker/kolektyw/dowody400m_ckpt/A_mini_R1 +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +0702c32c0ad602d8d12b16539e05cfbc68d1823cfd39b82b984357d78f27f664 /home/ubuntu/kolektyw/dowody400m/kod/ramie_v03.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +341aae81ce8512294dcd7de93ced26053025677b4ab449606be989f7fd898d27 /home/ubuntu/kolektyw/dowody400m/kod/kopiarka_ckpt.sh +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 diff --git a/test_wznowienia/A/run/srodowisko_resume0.txt b/test_wznowienia/A/run/srodowisko_resume0.txt new file mode 100644 index 0000000000000000000000000000000000000000..b9512da35b06a2236cf8342bbaf93278ff14b9ff --- /dev/null +++ b/test_wznowienia/A/run/srodowisko_resume0.txt @@ -0,0 +1,64 @@ +data 2026-10-23T02:33:05Z ARM R1 GPU 0 MINI 1 DATA_SEED 20261022 CKPT 20 RESUME 0 DEST /var/lib/docker/kolektyw/dowody400m_ckpt/A_mini_R1 +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +0702c32c0ad602d8d12b16539e05cfbc68d1823cfd39b82b984357d78f27f664 /home/ubuntu/kolektyw/dowody400m/kod/ramie_v03.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +341aae81ce8512294dcd7de93ced26053025677b4ab449606be989f7fd898d27 /home/ubuntu/kolektyw/dowody400m/kod/kopiarka_ckpt.sh +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 diff --git a/test_wznowienia/A/run/train.log b/test_wznowienia/A/run/train.log new file mode 100644 index 0000000000000000000000000000000000000000..7501c135db1d311cc93e1a4a889c7ca2c9577fac --- /dev/null +++ b/test_wznowienia/A/run/train.log @@ -0,0 +1,13 @@ +[04:33:11] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[04:33:11] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[04:33:13] model GPT-ref: 183.6M param (L16 d896 h14) +[04:33:14] optimizer=muon muon_lr=0.02 (mult=33.3x) +[04:33:14] r11 pamiec: act_ckpt=True loss_chunk=1024 +[04:33:14] r13: mixer_layout=AAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[04:33:15] torch.compile enabled +[04:33:15] START gpt-ref: steps=40 (od 0) lr=0.0006->6e-05 +[04:37:12] step 20/40 loss 10.3757 lr 2.40e-05 gnorm 2.72 44,100 tok/s peakVRAM 13.7GB +[04:37:14] ckpt @ 20 +[04:40:27] step 40/40 loss 9.5564 lr 4.80e-05 gnorm 1.29 53,978 tok/s peakVRAM 13.7GB +[04:40:29] ckpt @ 40 +[04:40:29] DONE diff --git a/test_wznowienia/A/run/train.out b/test_wznowienia/A/run/train.out new file mode 100644 index 0000000000000000000000000000000000000000..7501c135db1d311cc93e1a4a889c7ca2c9577fac --- /dev/null +++ b/test_wznowienia/A/run/train.out @@ -0,0 +1,13 @@ +[04:33:11] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[04:33:11] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[04:33:13] model GPT-ref: 183.6M param (L16 d896 h14) +[04:33:14] optimizer=muon muon_lr=0.02 (mult=33.3x) +[04:33:14] r11 pamiec: act_ckpt=True loss_chunk=1024 +[04:33:14] r13: mixer_layout=AAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[04:33:15] torch.compile enabled +[04:33:15] START gpt-ref: steps=40 (od 0) lr=0.0006->6e-05 +[04:37:12] step 20/40 loss 10.3757 lr 2.40e-05 gnorm 2.72 44,100 tok/s peakVRAM 13.7GB +[04:37:14] ckpt @ 20 +[04:40:27] step 40/40 loss 9.5564 lr 4.80e-05 gnorm 1.29 53,978 tok/s peakVRAM 13.7GB +[04:40:29] ckpt @ 40 +[04:40:29] DONE diff --git a/test_wznowienia/A/run/val_en.json b/test_wznowienia/A/run/val_en.json new file mode 100644 index 0000000000000000000000000000000000000000..70462fd5edffbe3caf15690a8e51d60eb4d9dcb7 --- /dev/null +++ b/test_wznowienia/A/run/val_en.json @@ -0,0 +1 @@ +{"schema": "val_bpb_v0", "ckpt_sha256": "f5877f84b95282b73d3b112299f14cd79aa28ca7a18d070ef91ee872eefb7d9c", "step": 40, "val_sha256": "046bc03b426b75f0e84db7cdf3965a9653d94e722b81b0d03fd0cb92884cf0c5", "tokenizer_sha256": "0640d3bd3674d7a6e59c540d945a2ad275a12e1bcaeb6007c37a90751f9815dd", "script_sha256": "ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1", "trainer_sha256": "db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b", "window": 4096, "n_windows": 547, "tokens": 2240512, "bytes": 9799872, "sum_nat": 19767938.0078125, "bpb": 2.9101508807943155, "ce_token": 8.822955649339303, "P": 183561103, "dtype": "fp32, tf32 off, bez autocast", "torch": "2.6.0+cu124", "device": "NVIDIA GeForce RTX 4090"} \ No newline at end of file diff --git a/test_wznowienia/A/run/val_en.out b/test_wznowienia/A/run/val_en.out new file mode 100644 index 0000000000000000000000000000000000000000..2ec0cb572a556d8b456a43ef4720a7c16f8740e2 --- /dev/null +++ b/test_wznowienia/A/run/val_en.out @@ -0,0 +1 @@ +VAL-BPB-OK step 40 bpb 2.91015 ce 8.82296 bajty 9799872 P 183561103 -> /home/ubuntu/kolektyw/dowody400m/test_resume/A/mini_R1/val_en.json diff --git a/test_wznowienia/A/run/val_pl.json b/test_wznowienia/A/run/val_pl.json new file mode 100644 index 0000000000000000000000000000000000000000..114411815836cef8a9f444d6523db21dd752300c --- /dev/null +++ b/test_wznowienia/A/run/val_pl.json @@ -0,0 +1 @@ +{"schema": "val_bpb_v0", "ckpt_sha256": "f5877f84b95282b73d3b112299f14cd79aa28ca7a18d070ef91ee872eefb7d9c", "step": 40, "val_sha256": "99e66265f47d249c72f5e76838aa23e82959b99089f720bea6e9ee2cb1f49ee1", "tokenizer_sha256": "0640d3bd3674d7a6e59c540d945a2ad275a12e1bcaeb6007c37a90751f9815dd", "script_sha256": "ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1", "trainer_sha256": "db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b", "window": 4096, "n_windows": 406, "tokens": 1662976, "bytes": 7234678, "sum_nat": 15573548.78125, "bpb": 3.105581422663419, "ce_token": 9.364866829858038, "P": 183561103, "dtype": "fp32, tf32 off, bez autocast", "torch": "2.6.0+cu124", "device": "NVIDIA GeForce RTX 4090"} \ No newline at end of file diff --git a/test_wznowienia/A/run/val_pl.out b/test_wznowienia/A/run/val_pl.out new file mode 100644 index 0000000000000000000000000000000000000000..23704dd159f6d401d757c71bf322deb9bd9d36c5 --- /dev/null +++ b/test_wznowienia/A/run/val_pl.out @@ -0,0 +1 @@ +VAL-BPB-OK step 40 bpb 3.10558 ce 9.36487 bajty 7234678 P 183561103 -> /home/ubuntu/kolektyw/dowody400m/test_resume/A/mini_R1/val_pl.json diff --git a/test_wznowienia/A/run/wynik.json b/test_wznowienia/A/run/wynik.json new file mode 100644 index 0000000000000000000000000000000000000000..67793435bace83e2faf4a1be5f6c30677f33b22b --- /dev/null +++ b/test_wznowienia/A/run/wynik.json @@ -0,0 +1 @@ +{"ramie": "R1", "ckpt_sha256": "f5877f84b95282b73d3b112299f14cd79aa28ca7a18d070ef91ee872eefb7d9c", "bpb_pl": 3.105581422663419, "bpb_en": 2.9101508807943155, "bpb_wazona_40_60": 2.988323097541957, "val_pl_sha256": "99e66265f47d249c72f5e76838aa23e82959b99089f720bea6e9ee2cb1f49ee1", "val_en_sha256": "046bc03b426b75f0e84db7cdf3965a9653d94e722b81b0d03fd0cb92884cf0c5", "P": 183561103, "step": 40} diff --git a/test_wznowienia/B/run/SHA256SUMS_kopiarka b/test_wznowienia/B/run/SHA256SUMS_kopiarka new file mode 100644 index 0000000000000000000000000000000000000000..6bfb48e438acb1535bff64a3b464591d7a56f691 --- /dev/null +++ b/test_wznowienia/B/run/SHA256SUMS_kopiarka @@ -0,0 +1 @@ +e41bdc94415849333814f1ae6a9b6a879c0695f2dadc18130c52830e595fce98 ckpt_40.pt diff --git a/test_wznowienia/B/run/args.txt b/test_wznowienia/B/run/args.txt new file mode 100644 index 0000000000000000000000000000000000000000..d8cd50fc5e5464187f275fa0a0b0672625a85e4f --- /dev/null +++ b/test_wznowienia/B/run/args.txt @@ -0,0 +1 @@ +--data-dir /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2 --out-dir /home/ubuntu/kolektyw/dowody400m/test_resume/B/mini_R1 --run-id dowody_R1 --vocab 32768 --dtype uint16 --block 4096 --n-layer 16 --n-embd 896 --n-head 14 --value-residual --batch 128 --grad-accum 8 --norm rmsnorm --pos rope --rope-theta 500000 --ffn swiglu --ffn-mult 2.667 --qk-norm --attn-bias 0 --optimizer muon --muon-lr 0.02 --lr 6e-4 --min-lr 6e-05 --wd 0.1 --schedule wsd --decay-frac 0.2 --warmup 500 --act-ckpt --loss-chunk 1024 --sampler random --seed 1 --data-seed 20261022 --compile --log-every 20 --eval-every 41 --eval-iters 1 --steps 40 --ckpt-every 20 \ No newline at end of file diff --git a/test_wznowienia/B/run/metrics.jsonl b/test_wznowienia/B/run/metrics.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..dab1509dc8249dc90f4a6513c25cc4a8bd2452ce --- /dev/null +++ b/test_wznowienia/B/run/metrics.jsonl @@ -0,0 +1 @@ +{"step": 40, "loss": 9.556369209289551, "lr": 4.7999999999999994e-05, "tok_s": 44341.98688205897} diff --git a/test_wznowienia/B/run/srodowisko_R1_mini1_resume1.txt b/test_wznowienia/B/run/srodowisko_R1_mini1_resume1.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a629c0cca81e95b5e213486ed7101d694994ff4 --- /dev/null +++ b/test_wznowienia/B/run/srodowisko_R1_mini1_resume1.txt @@ -0,0 +1,64 @@ +data 2026-10-23T02:42:10Z ARM R1 GPU 0 MINI 1 DATA_SEED 20261022 CKPT 20 RESUME 1 DEST /var/lib/docker/kolektyw/dowody400m_ckpt/B_mini_R1 +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +0702c32c0ad602d8d12b16539e05cfbc68d1823cfd39b82b984357d78f27f664 /home/ubuntu/kolektyw/dowody400m/kod/ramie_v03.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +341aae81ce8512294dcd7de93ced26053025677b4ab449606be989f7fd898d27 /home/ubuntu/kolektyw/dowody400m/kod/kopiarka_ckpt.sh +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 15082 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 14222 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 15482 MiB, 595.104.02 diff --git a/test_wznowienia/B/run/srodowisko_resume1.txt b/test_wznowienia/B/run/srodowisko_resume1.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a629c0cca81e95b5e213486ed7101d694994ff4 --- /dev/null +++ b/test_wznowienia/B/run/srodowisko_resume1.txt @@ -0,0 +1,64 @@ +data 2026-10-23T02:42:10Z ARM R1 GPU 0 MINI 1 DATA_SEED 20261022 CKPT 20 RESUME 1 DEST /var/lib/docker/kolektyw/dowody400m_ckpt/B_mini_R1 +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +0702c32c0ad602d8d12b16539e05cfbc68d1823cfd39b82b984357d78f27f664 /home/ubuntu/kolektyw/dowody400m/kod/ramie_v03.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +341aae81ce8512294dcd7de93ced26053025677b4ab449606be989f7fd898d27 /home/ubuntu/kolektyw/dowody400m/kod/kopiarka_ckpt.sh +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 15082 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 14222 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 15482 MiB, 595.104.02 diff --git a/test_wznowienia/B/run/train.log b/test_wznowienia/B/run/train.log new file mode 100644 index 0000000000000000000000000000000000000000..6e2ec5b7243cfbb7a4d05fe2a43547d2b845850c --- /dev/null +++ b/test_wznowienia/B/run/train.log @@ -0,0 +1,13 @@ +[04:42:16] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[04:42:16] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[04:42:18] model GPT-ref: 183.6M param (L16 d896 h14) +[04:42:19] optimizer=muon muon_lr=0.02 (mult=33.3x) +[04:42:19] r11 pamiec: act_ckpt=True loss_chunk=1024 +[04:42:19] r13: mixer_layout=AAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[04:42:20] torch.compile enabled +[04:42:20] RESUME @ 20 +[04:42:20] START gpt-ref: steps=40 (od 20) lr=0.0006->6e-05 +[04:42:20] train RNG fast-forward o 20 krokow (resume) +[04:46:17] step 40/40 loss 9.5564 lr 4.80e-05 gnorm 1.29 44,342 tok/s peakVRAM 14.5GB +[04:46:19] ckpt @ 40 +[04:46:19] DONE diff --git a/test_wznowienia/B/run/train.out b/test_wznowienia/B/run/train.out new file mode 100644 index 0000000000000000000000000000000000000000..6e2ec5b7243cfbb7a4d05fe2a43547d2b845850c --- /dev/null +++ b/test_wznowienia/B/run/train.out @@ -0,0 +1,13 @@ +[04:42:16] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[04:42:16] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[04:42:18] model GPT-ref: 183.6M param (L16 d896 h14) +[04:42:19] optimizer=muon muon_lr=0.02 (mult=33.3x) +[04:42:19] r11 pamiec: act_ckpt=True loss_chunk=1024 +[04:42:19] r13: mixer_layout=AAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[04:42:20] torch.compile enabled +[04:42:20] RESUME @ 20 +[04:42:20] START gpt-ref: steps=40 (od 20) lr=0.0006->6e-05 +[04:42:20] train RNG fast-forward o 20 krokow (resume) +[04:46:17] step 40/40 loss 9.5564 lr 4.80e-05 gnorm 1.29 44,342 tok/s peakVRAM 14.5GB +[04:46:19] ckpt @ 40 +[04:46:19] DONE diff --git a/test_wznowienia/B/run/val_en.json b/test_wznowienia/B/run/val_en.json new file mode 100644 index 0000000000000000000000000000000000000000..f897f3156f127fc50cfb168dfc973b4de978c24a --- /dev/null +++ b/test_wznowienia/B/run/val_en.json @@ -0,0 +1 @@ +{"schema": "val_bpb_v0", "ckpt_sha256": "e41bdc94415849333814f1ae6a9b6a879c0695f2dadc18130c52830e595fce98", "step": 40, "val_sha256": "046bc03b426b75f0e84db7cdf3965a9653d94e722b81b0d03fd0cb92884cf0c5", "tokenizer_sha256": "0640d3bd3674d7a6e59c540d945a2ad275a12e1bcaeb6007c37a90751f9815dd", "script_sha256": "ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1", "trainer_sha256": "db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b", "window": 4096, "n_windows": 547, "tokens": 2240512, "bytes": 9799872, "sum_nat": 19767949.125, "bpb": 2.9101525174188834, "ce_token": 8.822960611235288, "P": 183561103, "dtype": "fp32, tf32 off, bez autocast", "torch": "2.6.0+cu124", "device": "NVIDIA GeForce RTX 4090"} \ No newline at end of file diff --git a/test_wznowienia/B/run/val_en.out b/test_wznowienia/B/run/val_en.out new file mode 100644 index 0000000000000000000000000000000000000000..feb072884572bc4ba22792647b354adef134d9dc --- /dev/null +++ b/test_wznowienia/B/run/val_en.out @@ -0,0 +1 @@ +VAL-BPB-OK step 40 bpb 2.91015 ce 8.82296 bajty 9799872 P 183561103 -> /home/ubuntu/kolektyw/dowody400m/test_resume/B/mini_R1/val_en.json diff --git a/test_wznowienia/B/run/val_pl.json b/test_wznowienia/B/run/val_pl.json new file mode 100644 index 0000000000000000000000000000000000000000..2a3099eb5a2c9c381468a825964612c81375b009 --- /dev/null +++ b/test_wznowienia/B/run/val_pl.json @@ -0,0 +1 @@ +{"schema": "val_bpb_v0", "ckpt_sha256": "e41bdc94415849333814f1ae6a9b6a879c0695f2dadc18130c52830e595fce98", "step": 40, "val_sha256": "99e66265f47d249c72f5e76838aa23e82959b99089f720bea6e9ee2cb1f49ee1", "tokenizer_sha256": "0640d3bd3674d7a6e59c540d945a2ad275a12e1bcaeb6007c37a90751f9815dd", "script_sha256": "ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1", "trainer_sha256": "db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b", "window": 4096, "n_windows": 406, "tokens": 1662976, "bytes": 7234678, "sum_nat": 15573509.2890625, "bpb": 3.1055735473740054, "ce_token": 9.364843081958188, "P": 183561103, "dtype": "fp32, tf32 off, bez autocast", "torch": "2.6.0+cu124", "device": "NVIDIA GeForce RTX 4090"} \ No newline at end of file diff --git a/test_wznowienia/B/run/val_pl.out b/test_wznowienia/B/run/val_pl.out new file mode 100644 index 0000000000000000000000000000000000000000..ddf813fc2b90704956bf235fceb44e8aeeeeed8b --- /dev/null +++ b/test_wznowienia/B/run/val_pl.out @@ -0,0 +1 @@ +VAL-BPB-OK step 40 bpb 3.10557 ce 9.36484 bajty 7234678 P 183561103 -> /home/ubuntu/kolektyw/dowody400m/test_resume/B/mini_R1/val_pl.json diff --git a/test_wznowienia/B/run/wynik.json b/test_wznowienia/B/run/wynik.json new file mode 100644 index 0000000000000000000000000000000000000000..60e3eb78bd8b09907e7292e284e69a5c1ae47a3b --- /dev/null +++ b/test_wznowienia/B/run/wynik.json @@ -0,0 +1 @@ +{"ramie": "R1", "ckpt_sha256": "e41bdc94415849333814f1ae6a9b6a879c0695f2dadc18130c52830e595fce98", "bpb_pl": 3.1055735473740054, "bpb_en": 2.9101525174188834, "bpb_wazona_40_60": 2.988320929400932, "val_pl_sha256": "99e66265f47d249c72f5e76838aa23e82959b99089f720bea6e9ee2cb1f49ee1", "val_en_sha256": "046bc03b426b75f0e84db7cdf3965a9653d94e722b81b0d03fd0cb92884cf0c5", "P": 183561103, "step": 40} diff --git a/test_wznowienia/C/batch_sha.out b/test_wznowienia/C/batch_sha.out new file mode 100644 index 0000000000000000000000000000000000000000..7ff4ed5a6224a6945809512ca1ff1128e31cc979 --- /dev/null +++ b/test_wznowienia/C/batch_sha.out @@ -0,0 +1,12 @@ +[04:48:14] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[04:48:14] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[04:48:17] model GPT-ref: 183.6M param (L16 d896 h14) +[04:48:17] optimizer=muon muon_lr=0.02 (mult=33.3x) +[04:48:17] r11 pamiec: act_ckpt=True loss_chunk=1024 +[04:48:17] r13: mixer_layout=AAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[04:48:18] torch.compile enabled +[04:48:18] START gpt-ref: steps=40 (od 0) lr=0.0006->6e-05 +[04:48:18] BATCH-SHA krok biegu 20 = krok strumienia 20: c79fe35300cb3abc9ad3e84882962082a68ddb6952ffca83c6fddd20915fe624 +[04:48:18] BATCH-SHA krok biegu 21 = krok strumienia 21: 26e03a3af82c033293b73fd6e4562eea557800865ba82fbcab90ec5ae3a60f1d +[04:48:18] BATCH-SHA krok biegu 39 = krok strumienia 39: f19d235d8ae2a1d7e7022445901764efbbbda5e6626a113e566912bbd75fd254 +[04:48:18] BATCH-SHA DONE diff --git a/test_wznowienia/C/batch_sha.txt b/test_wznowienia/C/batch_sha.txt new file mode 100644 index 0000000000000000000000000000000000000000..4ccc1e923a7a596ba3ee4294770ed7d737a7e26e --- /dev/null +++ b/test_wznowienia/C/batch_sha.txt @@ -0,0 +1,3 @@ +BATCH-SHA krok biegu 20 = krok strumienia 20: c79fe35300cb3abc9ad3e84882962082a68ddb6952ffca83c6fddd20915fe624 +BATCH-SHA krok biegu 21 = krok strumienia 21: 26e03a3af82c033293b73fd6e4562eea557800865ba82fbcab90ec5ae3a60f1d +BATCH-SHA krok biegu 39 = krok strumienia 39: f19d235d8ae2a1d7e7022445901764efbbbda5e6626a113e566912bbd75fd254 diff --git a/test_wznowienia/C/train.log b/test_wznowienia/C/train.log new file mode 100644 index 0000000000000000000000000000000000000000..7ff4ed5a6224a6945809512ca1ff1128e31cc979 --- /dev/null +++ b/test_wznowienia/C/train.log @@ -0,0 +1,12 @@ +[04:48:14] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[04:48:14] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[04:48:17] model GPT-ref: 183.6M param (L16 d896 h14) +[04:48:17] optimizer=muon muon_lr=0.02 (mult=33.3x) +[04:48:17] r11 pamiec: act_ckpt=True loss_chunk=1024 +[04:48:17] r13: mixer_layout=AAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[04:48:18] torch.compile enabled +[04:48:18] START gpt-ref: steps=40 (od 0) lr=0.0006->6e-05 +[04:48:18] BATCH-SHA krok biegu 20 = krok strumienia 20: c79fe35300cb3abc9ad3e84882962082a68ddb6952ffca83c6fddd20915fe624 +[04:48:18] BATCH-SHA krok biegu 21 = krok strumienia 21: 26e03a3af82c033293b73fd6e4562eea557800865ba82fbcab90ec5ae3a60f1d +[04:48:18] BATCH-SHA krok biegu 39 = krok strumienia 39: f19d235d8ae2a1d7e7022445901764efbbbda5e6626a113e566912bbd75fd254 +[04:48:18] BATCH-SHA DONE diff --git a/test_wznowienia/D/batch_sha.out b/test_wznowienia/D/batch_sha.out new file mode 100644 index 0000000000000000000000000000000000000000..84196b31d12939b0d183d3b5328883b1a7edd823 --- /dev/null +++ b/test_wznowienia/D/batch_sha.out @@ -0,0 +1,14 @@ +[04:48:21] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[04:48:21] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[04:48:23] model GPT-ref: 183.6M param (L16 d896 h14) +[04:48:24] optimizer=muon muon_lr=0.02 (mult=33.3x) +[04:48:24] r11 pamiec: act_ckpt=True loss_chunk=1024 +[04:48:24] r13: mixer_layout=AAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[04:48:24] torch.compile enabled +[04:48:25] RESUME @ 20 +[04:48:25] START gpt-ref: steps=40 (od 20) lr=0.0006->6e-05 +[04:48:25] train RNG fast-forward o 20 krokow (resume) +[04:48:25] BATCH-SHA krok biegu 20 = krok strumienia 20: c79fe35300cb3abc9ad3e84882962082a68ddb6952ffca83c6fddd20915fe624 +[04:48:25] BATCH-SHA krok biegu 21 = krok strumienia 21: 26e03a3af82c033293b73fd6e4562eea557800865ba82fbcab90ec5ae3a60f1d +[04:48:25] BATCH-SHA krok biegu 39 = krok strumienia 39: f19d235d8ae2a1d7e7022445901764efbbbda5e6626a113e566912bbd75fd254 +[04:48:25] BATCH-SHA DONE diff --git a/test_wznowienia/D/batch_sha.txt b/test_wznowienia/D/batch_sha.txt new file mode 100644 index 0000000000000000000000000000000000000000..4ccc1e923a7a596ba3ee4294770ed7d737a7e26e --- /dev/null +++ b/test_wznowienia/D/batch_sha.txt @@ -0,0 +1,3 @@ +BATCH-SHA krok biegu 20 = krok strumienia 20: c79fe35300cb3abc9ad3e84882962082a68ddb6952ffca83c6fddd20915fe624 +BATCH-SHA krok biegu 21 = krok strumienia 21: 26e03a3af82c033293b73fd6e4562eea557800865ba82fbcab90ec5ae3a60f1d +BATCH-SHA krok biegu 39 = krok strumienia 39: f19d235d8ae2a1d7e7022445901764efbbbda5e6626a113e566912bbd75fd254 diff --git a/test_wznowienia/D/train.log b/test_wznowienia/D/train.log new file mode 100644 index 0000000000000000000000000000000000000000..84196b31d12939b0d183d3b5328883b1a7edd823 --- /dev/null +++ b/test_wznowienia/D/train.log @@ -0,0 +1,14 @@ +[04:48:21] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[04:48:21] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[04:48:23] model GPT-ref: 183.6M param (L16 d896 h14) +[04:48:24] optimizer=muon muon_lr=0.02 (mult=33.3x) +[04:48:24] r11 pamiec: act_ckpt=True loss_chunk=1024 +[04:48:24] r13: mixer_layout=AAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[04:48:24] torch.compile enabled +[04:48:25] RESUME @ 20 +[04:48:25] START gpt-ref: steps=40 (od 20) lr=0.0006->6e-05 +[04:48:25] train RNG fast-forward o 20 krokow (resume) +[04:48:25] BATCH-SHA krok biegu 20 = krok strumienia 20: c79fe35300cb3abc9ad3e84882962082a68ddb6952ffca83c6fddd20915fe624 +[04:48:25] BATCH-SHA krok biegu 21 = krok strumienia 21: 26e03a3af82c033293b73fd6e4562eea557800865ba82fbcab90ec5ae3a60f1d +[04:48:25] BATCH-SHA krok biegu 39 = krok strumienia 39: f19d235d8ae2a1d7e7022445901764efbbbda5e6626a113e566912bbd75fd254 +[04:48:25] BATCH-SHA DONE diff --git a/test_wznowienia/mini_R1/run/args.txt b/test_wznowienia/mini_R1/run/args.txt new file mode 100644 index 0000000000000000000000000000000000000000..32dece764167003a3ddec83b8b28cf867c7704b8 --- /dev/null +++ b/test_wznowienia/mini_R1/run/args.txt @@ -0,0 +1 @@ +--data-dir /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2 --out-dir /home/ubuntu/kolektyw/dowody400m/wyniki/mini_R1 --run-id dowody_R1 --vocab 32768 --dtype uint16 --block 4096 --n-layer 16 --n-embd 896 --n-head 14 --value-residual --batch 128 --grad-accum 16 --norm rmsnorm --pos rope --rope-theta 500000 --ffn swiglu --ffn-mult 2.667 --qk-norm --attn-bias 0 --optimizer muon --muon-lr 0.02 --lr 6e-4 --min-lr 6e-05 --wd 0.1 --schedule wsd --decay-frac 0.2 --warmup 500 --act-ckpt --loss-chunk 1024 --sampler random --seed 1 --data-seed 20261022 --compile --log-every 20 --eval-every 21 --eval-iters 1 --steps 20 --ckpt-every 20 \ No newline at end of file diff --git a/test_wznowienia/mini_R1/run/metrics.jsonl b/test_wznowienia/mini_R1/run/metrics.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..fd65bd01fd35deb62525b97d2c8fabc0c7556eb6 --- /dev/null +++ b/test_wznowienia/mini_R1/run/metrics.jsonl @@ -0,0 +1 @@ +{"step": 20, "loss": 10.37570834159851, "lr": 2.3999999999999997e-05, "tok_s": 40836.1162943951} diff --git a/test_wznowienia/mini_R1/run/srodowisko.txt b/test_wznowienia/mini_R1/run/srodowisko.txt new file mode 100644 index 0000000000000000000000000000000000000000..0b12d2d65188eb478062eb0b197f8d961143b9e7 --- /dev/null +++ b/test_wznowienia/mini_R1/run/srodowisko.txt @@ -0,0 +1,63 @@ +data 2026-10-22T11:48:41Z ARM R1 GPU 0 MINI 1 DATA_SEED 20261022 +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +a607076bd6e11cad879b8213b37492a4b4f5cbf8fa96205b7ebaeace8a25213e /home/ubuntu/kolektyw/dowody400m/kod/ramie.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 diff --git a/test_wznowienia/mini_R1/run/srodowisko_R1_mini0.txt b/test_wznowienia/mini_R1/run/srodowisko_R1_mini0.txt new file mode 100644 index 0000000000000000000000000000000000000000..e7e2dd3ec25f8f96cde2b26a7eaf5cd13a428e7c --- /dev/null +++ b/test_wznowienia/mini_R1/run/srodowisko_R1_mini0.txt @@ -0,0 +1,63 @@ +data 2026-10-22T11:59:49Z ARM R1 GPU 0 MINI 0 DATA_SEED 20261022 +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +a607076bd6e11cad879b8213b37492a4b4f5cbf8fa96205b7ebaeace8a25213e /home/ubuntu/kolektyw/dowody400m/kod/ramie.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 diff --git a/test_wznowienia/mini_R1/run/srodowisko_R1_mini1.txt b/test_wznowienia/mini_R1/run/srodowisko_R1_mini1.txt new file mode 100644 index 0000000000000000000000000000000000000000..0b12d2d65188eb478062eb0b197f8d961143b9e7 --- /dev/null +++ b/test_wznowienia/mini_R1/run/srodowisko_R1_mini1.txt @@ -0,0 +1,63 @@ +data 2026-10-22T11:48:41Z ARM R1 GPU 0 MINI 1 DATA_SEED 20261022 +CPATH=/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include/x86_64-linux-gnu/python3.12:/home/ubuntu/kolektyw/polocr/pydev/root/usr/include +a607076bd6e11cad879b8213b37492a4b4f5cbf8fa96205b7ebaeace8a25213e /home/ubuntu/kolektyw/dowody400m/kod/ramie.sh +db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b /home/ubuntu/kolektyw/dowody400m/kod/train_gpt_ref_r23.py +ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1 /home/ubuntu/kolektyw/dowody400m/kod/val_bpb.py +61859a5b32d286a904dc5a8a1998a2a2f986f8b80366868815b32794a02dfa11 /home/ubuntu/kolektyw/dane_dowody/out_v1_fw2/SHA256SUMS +2.6.0+cu124 12.4 +accelerate==1.15.0 +annotated-doc==0.0.5 +anyio==4.15.1 +certifi==2026.10.19 +click==8.5.0 +filelock==4.2.4 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.7.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.33.0 +idna==3.20 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.2.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +packaging==26.3 +pillow==12.3.0 +psutil==7.2.2 +Pygments==2.21.0 +PyYAML==6.0.3 +regex==2026.9.29 +rich==15.0.0 +safetensors==0.8.0 +setuptools==84.2.0 +shellingham==1.5.4 +sympy==1.13.1 +tokenizers==0.23.3 +torch==2.6.0+cu124 +torchvision==0.21.0+cu124 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.2.0 +typer==0.27.3 +typing_extensions==4.16.0 +0, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +1, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +2, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 +3, NVIDIA GeForce RTX 4090, 1 MiB, 595.104.02 diff --git a/test_wznowienia/mini_R1/run/train.log b/test_wznowienia/mini_R1/run/train.log new file mode 100644 index 0000000000000000000000000000000000000000..c288eb080cd046836c5ee444be28466bf874cf2f --- /dev/null +++ b/test_wznowienia/mini_R1/run/train.log @@ -0,0 +1,11 @@ +[13:48:47] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[13:48:47] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[13:48:49] model GPT-ref: 183.6M param (L16 d896 h14) +[13:48:50] optimizer=muon muon_lr=0.02 (mult=33.3x) +[13:48:50] r11 pamiec: act_ckpt=True loss_chunk=1024 +[13:48:50] r13: mixer_layout=AAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[13:48:51] torch.compile enabled +[13:48:51] START gpt-ref: steps=20 (od 0) lr=0.0006->6e-05 +[13:53:07] step 20/20 loss 10.3757 lr 2.40e-05 gnorm 2.72 40,836 tok/s peakVRAM 8.2GB +[13:53:09] ckpt @ 20 +[13:53:09] DONE diff --git a/test_wznowienia/mini_R1/run/train.out b/test_wznowienia/mini_R1/run/train.out new file mode 100644 index 0000000000000000000000000000000000000000..c288eb080cd046836c5ee444be28466bf874cf2f --- /dev/null +++ b/test_wznowienia/mini_R1/run/train.out @@ -0,0 +1,11 @@ +[13:48:47] device=cuda:0 bf16=True dev=NVIDIA GeForce RTX 4090 world=1 +[13:48:47] dane: train=2,500,001,211B block=4096 batch=128 tok/step=524,288 +[13:48:49] model GPT-ref: 183.6M param (L16 d896 h14) +[13:48:50] optimizer=muon muon_lr=0.02 (mult=33.3x) +[13:48:50] r11 pamiec: act_ckpt=True loss_chunk=1024 +[13:48:50] r13: mixer_layout=AAAAAAAAAAAAAAAA gdn_head_dim=64 attn_bias=0 rope_style=interleaved +[13:48:51] torch.compile enabled +[13:48:51] START gpt-ref: steps=20 (od 0) lr=0.0006->6e-05 +[13:53:07] step 20/20 loss 10.3757 lr 2.40e-05 gnorm 2.72 40,836 tok/s peakVRAM 8.2GB +[13:53:09] ckpt @ 20 +[13:53:09] DONE diff --git a/test_wznowienia/mini_R1/run/val_en.json b/test_wznowienia/mini_R1/run/val_en.json new file mode 100644 index 0000000000000000000000000000000000000000..373c437eaaa85710af45d005ee8d80f4e307296b --- /dev/null +++ b/test_wznowienia/mini_R1/run/val_en.json @@ -0,0 +1 @@ +{"schema": "val_bpb_v0", "ckpt_sha256": "1c501300ddaaee8bdc063e42db798a394087f6169a3e84446c5346474cbb6136", "step": 20, "val_sha256": "046bc03b426b75f0e84db7cdf3965a9653d94e722b81b0d03fd0cb92884cf0c5", "tokenizer_sha256": "0640d3bd3674d7a6e59c540d945a2ad275a12e1bcaeb6007c37a90751f9815dd", "script_sha256": "ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1", "trainer_sha256": "db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b", "window": 4096, "n_windows": 547, "tokens": 2240512, "bytes": 9799872, "sum_nat": 22177769.9375, "bpb": 3.2649159812299167, "ce_token": 9.898527630068484, "P": 183561103, "dtype": "fp32, tf32 off, bez autocast", "torch": "2.6.0+cu124", "device": "NVIDIA GeForce RTX 4090"} \ No newline at end of file diff --git a/test_wznowienia/mini_R1/run/val_en.out b/test_wznowienia/mini_R1/run/val_en.out new file mode 100644 index 0000000000000000000000000000000000000000..8b412c7928b05d4ebedd47110ec26b0b7eebc93f --- /dev/null +++ b/test_wznowienia/mini_R1/run/val_en.out @@ -0,0 +1 @@ +VAL-BPB-OK step 20 bpb 3.26492 ce 9.89853 bajty 9799872 P 183561103 -> /home/ubuntu/kolektyw/dowody400m/wyniki/mini_R1/val_en.json diff --git a/test_wznowienia/mini_R1/run/val_pl.json b/test_wznowienia/mini_R1/run/val_pl.json new file mode 100644 index 0000000000000000000000000000000000000000..49d447ddcc3a430d3526a079e8542fb86b6158ea --- /dev/null +++ b/test_wznowienia/mini_R1/run/val_pl.json @@ -0,0 +1 @@ +{"schema": "val_bpb_v0", "ckpt_sha256": "1c501300ddaaee8bdc063e42db798a394087f6169a3e84446c5346474cbb6136", "step": 20, "val_sha256": "99e66265f47d249c72f5e76838aa23e82959b99089f720bea6e9ee2cb1f49ee1", "tokenizer_sha256": "0640d3bd3674d7a6e59c540d945a2ad275a12e1bcaeb6007c37a90751f9815dd", "script_sha256": "ce84287cbdea65e02ce8f5b4af2617aae0b8239db4984c456cee8f75a6866ab1", "trainer_sha256": "db36c150e95c184e82b5f84c3ee783fd5fd2c62e960a438177f543dc3eba1d0b", "window": 4096, "n_windows": 406, "tokens": 1662976, "bytes": 7234678, "sum_nat": 16791464.5390625, "bpb": 3.348450700883765, "ce_token": 10.097238047369595, "P": 183561103, "dtype": "fp32, tf32 off, bez autocast", "torch": "2.6.0+cu124", "device": "NVIDIA GeForce RTX 4090"} \ No newline at end of file diff --git a/test_wznowienia/mini_R1/run/val_pl.out b/test_wznowienia/mini_R1/run/val_pl.out new file mode 100644 index 0000000000000000000000000000000000000000..ae035659d57882fe7c6afca21eaf328272f7eeec --- /dev/null +++ b/test_wznowienia/mini_R1/run/val_pl.out @@ -0,0 +1 @@ +VAL-BPB-OK step 20 bpb 3.34845 ce 10.09724 bajty 7234678 P 183561103 -> /home/ubuntu/kolektyw/dowody400m/wyniki/mini_R1/val_pl.json diff --git a/test_wznowienia/mini_R1/run/wynik.json b/test_wznowienia/mini_R1/run/wynik.json new file mode 100644 index 0000000000000000000000000000000000000000..263a5bf80855f3f2ada1611cbc15dc35ec7af5b3 --- /dev/null +++ b/test_wznowienia/mini_R1/run/wynik.json @@ -0,0 +1 @@ +{"ramie": "R1", "ckpt_sha256": "1c501300ddaaee8bdc063e42db798a394087f6169a3e84446c5346474cbb6136", "bpb_pl": 3.348450700883765, "bpb_en": 3.2649159812299167, "bpb_wazona_40_60": 3.298329869091456, "val_pl_sha256": "99e66265f47d249c72f5e76838aa23e82959b99089f720bea6e9ee2cb1f49ee1", "val_en_sha256": "046bc03b426b75f0e84db7cdf3965a9653d94e722b81b0d03fd0cb92884cf0c5", "P": 183561103, "step": 20} diff --git a/test_wznowienia/test.log b/test_wznowienia/test.log new file mode 100644 index 0000000000000000000000000000000000000000..100e6fd4aeaa96c211ea9dd11eebe46d90adcff4 --- /dev/null +++ b/test_wznowienia/test.log @@ -0,0 +1,20 @@ +TEST-START 2026-10-23T02:33:05Z GPU 0 +RAMIE R1 2026-10-23T02:33:09Z gpu 0 steps 40 mb 16 ga 8 seed 1 data_seed 20261022 lr 6e-4 min_lr 6e-05 mini 1 ckpt_every 20 resume 0 +RAMIE-OK R1 2026-10-23T02:41:58Z ckpt f5877f84 {"ramie": "R1", "ckpt_sha256": "f5877f84b95282b73d3b112299f14cd79aa28ca7a18d070ef91ee872eefb7d9c", "bpb_pl": 3.105581422663419, "bpb_en": 2.9101508807943155, "bpb_wazona_40_60": 2.988323097541957, "val_pl_sha256": "99e66265f47d249c72f5e76838aa23e82959b99089f720bea6e9ee2cb1f49ee1", "val_en_sha256": "046bc03b426b75f0e84db7cdf3965a9653d94e722b81b0d03fd0cb92884cf0c5", "P": 183561103, "step": 40} +RAMIE R1 2026-10-23T02:42:15Z gpu 0 steps 40 mb 16 ga 8 seed 1 data_seed 20261022 lr 6e-4 min_lr 6e-05 mini 1 ckpt_every 20 resume 1 +RAMIE-OK R1 2026-10-23T02:47:48Z ckpt e41bdc94 {"ramie": "R1", "ckpt_sha256": "e41bdc94415849333814f1ae6a9b6a879c0695f2dadc18130c52830e595fce98", "bpb_pl": 3.1055735473740054, "bpb_en": 2.9101525174188834, "bpb_wazona_40_60": 2.988320929400932, "val_pl_sha256": "99e66265f47d249c72f5e76838aa23e82959b99089f720bea6e9ee2cb1f49ee1", "val_en_sha256": "046bc03b426b75f0e84db7cdf3965a9653d94e722b81b0d03fd0cb92884cf0c5", "P": 183561103, "step": 40} +== sha batchy (C ciągły | D wznowiony) +BATCH-SHA krok biegu 20 = krok strumienia 20: c79fe35300cb3abc9ad3e84882962082a68ddb6952ffca83c6fddd20915fe624 BATCH-SHA krok biegu 20 = krok strumienia 20: c79fe35300cb3abc9ad3e84882962082a68ddb6952ffca83c6fddd20915fe624 +BATCH-SHA krok biegu 21 = krok strumienia 21: 26e03a3af82c033293b73fd6e4562eea557800865ba82fbcab90ec5ae3a60f1d BATCH-SHA krok biegu 21 = krok strumienia 21: 26e03a3af82c033293b73fd6e4562eea557800865ba82fbcab90ec5ae3a60f1d +BATCH-SHA krok biegu 39 = krok strumienia 39: f19d235d8ae2a1d7e7022445901764efbbbda5e6626a113e566912bbd75fd254 BATCH-SHA krok biegu 39 = krok strumienia 39: f19d235d8ae2a1d7e7022445901764efbbbda5e6626a113e566912bbd75fd254 +== straty +/home/ubuntu/kolektyw/dowody400m/test_resume/A/mini_R1/train.log:[04:37:12] step 20/40 loss 10.3757 lr 2.40e-05 gnorm 2.72 44,100 tok/s peakVRAM 13.7GB +/home/ubuntu/kolektyw/dowody400m/test_resume/A/mini_R1/train.log:[04:40:27] step 40/40 loss 9.5564 lr 4.80e-05 gnorm 1.29 53,978 tok/s peakVRAM 13.7GB +/home/ubuntu/kolektyw/dowody400m/test_resume/B/mini_R1/train.log:[04:46:17] step 40/40 loss 9.5564 lr 4.80e-05 gnorm 1.29 44,342 tok/s peakVRAM 14.5GB +== wynik A +{"ramie": "R1", "ckpt_sha256": "f5877f84b95282b73d3b112299f14cd79aa28ca7a18d070ef91ee872eefb7d9c", "bpb_pl": 3.105581422663419, "bpb_en": 2.9101508807943155, "bpb_wazona_40_60": 2.988323097541957, "val_pl_sha256": "99e66265f47d249c72f5e76838aa23e82959b99089f720bea6e9ee2cb1f49ee1", "val_en_sha256": "046bc03b426b75f0e84db7cdf3965a9653d94e722b81b0d03fd0cb92884cf0c5", "P": 183561103, "step": 40} + +== wynik B +{"ramie": "R1", "ckpt_sha256": "e41bdc94415849333814f1ae6a9b6a879c0695f2dadc18130c52830e595fce98", "bpb_pl": 3.1055735473740054, "bpb_en": 2.9101525174188834, "bpb_wazona_40_60": 2.988320929400932, "val_pl_sha256": "99e66265f47d249c72f5e76838aa23e82959b99089f720bea6e9ee2cb1f49ee1", "val_en_sha256": "046bc03b426b75f0e84db7cdf3965a9653d94e722b81b0d03fd0cb92884cf0c5", "P": 183561103, "step": 40} + +TEST-WZNOWIENIA-OK 2026-10-23T02:48:25Z