coeiico's picture
Add every run in its own folder and the model card
4a570eb verified
Raw History Blame Contribute Delete
6.08 kB
{
"config": {
"run_name": "default-seed1",
"seed": 1,
"quick": false,
"device": "auto",
"eval_every": 250,
"task": {
"n_values": 64,
"length": 16
},
"splits": {
"val_seed": 1000003,
"val_size": 1024,
"test_seed": 2000003,
"test_size": 2048
},
"model": {
"n_layers": 4,
"d_model": 256,
"n_heads": 8,
"d_mlp": 1024,
"mlp": "gelu",
"norm": "layernorm",
"dropout": 0.0
},
"optim": {
"steps": 4000,
"batch_size": 256,
"lr": 0.001,
"warmup_steps": 100,
"min_lr_ratio": 0.1,
"weight_decay": 0.1,
"grad_clip": 1.0
}
},
"model_config": {
"vocab_size": 65,
"n_layers": 4,
"d_model": 256,
"n_heads": 8,
"d_mlp": 1024,
"max_seq_len": 33,
"mlp": "gelu",
"norm": "layernorm",
"dropout": 0.0
},
"parameters": 3201280,
"data": {
"task": "sorting",
"version": 1,
"n_values": 64,
"length": 16,
"val_seed": 1000003,
"val_size": 1024,
"test_seed": 2000003,
"test_size": 2048,
"train_seed": 1
},
"code": {
"commit": "19276f75392bee9d5f6f61820395002a0746e349",
"dirty": true,
"snapshot": "source-snapshot.tar.gz"
},
"environment": {
"python": "3.12.3",
"torch": "2.15.0+cu132",
"device": "cuda",
"precision": "torch.float32",
"uv_lock_sha256": "60eba889e8dcd90d00fdcc282a51fc42df3869f3ef3807b1946aaa3cfd5f62e2",
"gpu": "NVIDIA GeForce RTX 3060",
"cuda": "13.2"
},
"resumable": false,
"status": "finished",
"seconds": 646.6,
"history": [
{
"step": 250,
"train_loss": 0.06796592473983765,
"val_loss": 0.08403412997722626,
"val_token_accuracy": 0.97271728515625,
"val_exact_match": 0.650390625,
"grad_norm": 2.1865394115448,
"lr": 0.0009967189933441244
},
{
"step": 500,
"train_loss": 0.03714346885681152,
"val_loss": 0.044457193464040756,
"val_token_accuracy": 0.98626708984375,
"val_exact_match": 0.8046875,
"grad_norm": 1.7170097827911377,
"lr": 0.0009768413988762155
},
{
"step": 750,
"train_loss": 0.03594381362199783,
"val_loss": 0.028060290962457657,
"val_token_accuracy": 0.99298095703125,
"val_exact_match": 0.900390625,
"grad_norm": 2.0198495388031006,
"lr": 0.0009397114317029974
},
{
"step": 1000,
"train_loss": 0.008170297369360924,
"val_loss": 0.012762336991727352,
"val_token_accuracy": 0.9959716796875,
"val_exact_match": 0.94140625,
"grad_norm": 0.43693360686302185,
"lr": 0.0008868298366769955
},
{
"step": 1250,
"train_loss": 0.006513639353215694,
"val_loss": 0.018579615280032158,
"val_token_accuracy": 0.99481201171875,
"val_exact_match": 0.9208984375,
"grad_norm": 0.49892523884773254,
"lr": 0.0008203340189070906
},
{
"step": 1500,
"train_loss": 0.008571112528443336,
"val_loss": 0.00933537445962429,
"val_token_accuracy": 0.997802734375,
"val_exact_match": 0.9658203125,
"grad_norm": 0.7041215896606445,
"lr": 0.0007429116526313744
},
{
"step": 1750,
"train_loss": 0.008580565452575684,
"val_loss": 0.011725695803761482,
"val_token_accuracy": 0.997314453125,
"val_exact_match": 0.9580078125,
"grad_norm": 0.5222469568252563,
"lr": 0.0006576920489294011
},
{
"step": 2000,
"train_loss": 0.0038308706134557724,
"val_loss": 0.003425079397857189,
"val_token_accuracy": 0.9991455078125,
"val_exact_match": 0.986328125,
"grad_norm": 0.217371866106987,
"lr": 0.0005681196730492369
},
{
"step": 2250,
"train_loss": 0.002150578424334526,
"val_loss": 0.001471434603445232,
"val_token_accuracy": 0.9996337890625,
"val_exact_match": 0.994140625,
"grad_norm": 0.4117131531238556,
"lr": 0.00047781492361400794
},
{
"step": 2500,
"train_loss": 0.0023180556017905474,
"val_loss": 0.0010354401310905814,
"val_token_accuracy": 0.9996337890625,
"val_exact_match": 0.994140625,
"grad_norm": 0.2440403699874878,
"lr": 0.0003904278008308589
},
{
"step": 2750,
"train_loss": 0.0008889065939001739,
"val_loss": 0.000499909685458988,
"val_token_accuracy": 0.9998779296875,
"val_exact_match": 0.998046875,
"grad_norm": 0.18215815722942352,
"lr": 0.0003094903782424895
},
{
"step": 3000,
"train_loss": 0.00015005430032033473,
"val_loss": 0.00048269086983054876,
"val_token_accuracy": 0.99993896484375,
"val_exact_match": 0.9990234375,
"grad_norm": 0.08666612952947617,
"lr": 0.0002382740409206803
},
{
"step": 3250,
"train_loss": 0.0008362794178538024,
"val_loss": 0.000332412077113986,
"val_token_accuracy": 0.99993896484375,
"val_exact_match": 0.9990234375,
"grad_norm": 0.2885400354862213,
"lr": 0.00017965726034785466
},
{
"step": 3500,
"train_loss": 0.001066122087650001,
"val_loss": 2.008096817007754e-05,
"val_token_accuracy": 1.0,
"val_exact_match": 1.0,
"grad_norm": 0.37905609607696533,
"lr": 0.00013600925035352912
},
{
"step": 3750,
"train_loss": 1.979812077479437e-05,
"val_loss": 1.038504888128955e-05,
"val_token_accuracy": 1.0,
"val_exact_match": 1.0,
"grad_norm": 0.003974005114287138,
"lr": 0.00010909420658097956
},
{
"step": 4000,
"train_loss": 5.5594000514247455e-06,
"val_loss": 0.00019783558673225343,
"val_token_accuracy": 0.9998779296875,
"val_exact_match": 0.998046875,
"grad_norm": 0.0021852694917470217,
"lr": 0.0001
}
],
"final": {
"val_loss": 0.00019783558673225343,
"val_token_accuracy": 0.9998779296875,
"val_exact_match": 0.998046875
}
}